Release v0.11.1

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

.gitea/workflows/release.yml

@@ -93,6 +93,24 @@ jobs:
           ./dist/llmgw-${version}-linux-amd64 --version
           cat dist/checksums.txt
 
+      # Signed when the repository has a signing key, so a machine that trusts
+      # it installs nothing the server alone could have put there. Without the
+      # secret nothing is signed and releases are checked on their checksums,
+      # as before; a build that trusts a key refuses those, so the secret has
+      # to be in place before trusted.pub is filled in.
+      - name: Sign the checksums
+        env:
+          RELEASE_SIGNING_KEY: ${{ secrets.RELEASE_SIGNING_KEY }}
+        run: |
+          set -euo pipefail
+          version="${GITHUB_REF_NAME:-$(git describe --tags --always)}"
+          if [ -z "${RELEASE_SIGNING_KEY:-}" ]; then
+            echo "no RELEASE_SIGNING_KEY secret; releases are not signed"
+            exit 0
+          fi
+          ./dist/llmgw-${version}-linux-amd64 release sign dist/checksums.txt
+          echo "signed: dist/checksums.txt.sig"
+
       - name: Attach to the release
         env:
           TOKEN: ${{ secrets.GITEA_TOKEN }}

README.md

@@ -1,7 +1,7 @@
 # llmgw
 
-A single-binary LLM gateway for clusters with multiple InfiniBand fabrics and
-heterogeneous GPUs.
+A single-binary LLM gateway for clusters with multiple InfiniBand fabrics,
+NVLink domains and heterogeneous GPUs.
 
 It serves LLMs running on your own GPU servers (vLLM / SGLang) behind one
 OpenAI-compatible endpoint. Design document: [llmgw-plan.md](llmgw-plan.md).
@@ -49,15 +49,23 @@ a header would let any client promote itself into the top priority band.
 
 One thing is worth saying plainly:
 multi-node replicas are proven on a simulated cluster and on real hardware
-without a fabric. The InfiniBand, NVML and NCCL paths are written against the
-formats and failure modes those tools actually have, but they have not yet run
-on a machine that has them.
+without a fabric. The InfiniBand, NVLink and NCCL paths — telling fabrics apart
+by their subnet managers, mapping them, finding NVLink domains, launching across
+nodes — are written against the formats and failure modes those tools actually
+have and tested against them, but they have not yet run on a machine that has
+them. On such a machine, run `llmgw doctor --json` first and keep the report;
+[docs/fabrics.md](docs/fabrics.md) says what to look at after that.
 
 ## Documentation
 
-- [docs/install.md](docs/install.md) — installing from nothing: control plane,
-  nodes, engines, weights, first request
-- [docs/quickstart.md](docs/quickstart.md) — the short path
+- [docs/quick-install.md](docs/quick-install.md) — installing on servers, in
+  six steps: start here
+- [docs/install.md](docs/install.md) — the same from nothing, with every option
+  explained: control plane, nodes, engines, weights, first request
+- [docs/quickstart.md](docs/quickstart.md) — one workstation, from source
+- [docs/fabrics.md](docs/fabrics.md) — InfiniBand fabrics and NVLink: what
+  llmgw needs of them, checking a node with `llmgw doctor`, and what to look at
+  when a replica spanning nodes will not run
 - [docs/choosing-models.md](docs/choosing-models.md) — which models to run,
   how many of each, and where the weights should live
 - [docs/data.md](docs/data.md) — what llmgw stores, what it does not, and
@@ -91,6 +99,7 @@ llmgw ctl replicas --model llama     # replicas and their shards
 llmgw ctl explain-placement llama    # why a model has not been placed
 llmgw ctl drain gpu-a-01             # move work off a node, one replica at a time
 llmgw ctl cordon gpu-a-01            # stop new placement, leave what runs
+sudo llmgw doctor                    # on a node: what its GPUs, NVLink and fabric need
 ```
 
 `ctl` talks to the admin port (`:9090` by default), not the client-facing one.

api/proto/llmgw/v1/agent.proto

@@ -24,6 +24,9 @@ message AgentMessage {
     Heartbeat heartbeat = 2;
     ShardStatus shard = 3;
     LogsReply logs = 4;
+    FabricScanReply fabric_scan = 5;
+    FabricTestReply fabric_test = 6;
+    UpgradeReply upgrade = 7;
   }
 }
 
@@ -37,9 +40,21 @@ message ControlMessage {
     RemoveWeights remove_weights = 5;
     Provision provision = 6;
     LogsRequest logs = 7;
+    FabricScanRequest fabric_scan = 8;
+    FabricTestRequest fabric_test = 9;
+    UpgradeAgent upgrade = 10;
+    ControlPlanes control_planes = 11;
   }
 }
 
+// ControlPlanes is every control plane an agent may dial, sent by the leader
+// when an agent registers and whenever the list changes. The agent keeps it
+// beside its configuration, so a second control plane added after the node
+// was onboarded is one it fails over to without anybody editing the node.
+message ControlPlanes {
+  repeated string servers = 1;
+}
+
 // Register is the first message on every stream. It carries the node's full
 // hardware inventory, so the control plane needs no separate discovery call and
 // a reconnecting agent automatically refreshes what the scheduler knows.
@@ -114,6 +129,25 @@ message Register {
   // meant "an agent too old to say" and constrained nothing — so a freshly
   // built node with no engine installed yet was a candidate for every model.
   bool engines_probed = 14;
+  // cpu_count is the CPUs the agent may run on. Placement reserves cores
+  // for CPU replicas out of it, the way it reserves memory out of
+  // memory_total_mb; zero, from an older agent, reserves nothing.
+  int32 cpu_count = 18;
+  // imex is the service that lets a multi-node NVLink domain share memory.
+  IMEX imex = 15;
+
+  // installable_engines are the engines this agent knows how to install
+  // itself. The control plane asks it for no others. Empty is an agent older
+  // than the field, which could install llama.cpp and nothing else: asked for
+  // vLLM, it answered "unsupported", and every node of a cluster half way
+  // through an upgrade showed as failing.
+  repeated string installable_engines = 16;
+
+  // features are the requests this agent answers beyond the ones every agent
+  // does: "fabric-test". An agent that does not know a request ignores it, so
+  // without this the control plane would wait minutes for an answer that is
+  // never coming.
+  repeated string features = 17;
 }
 
 // Host describes the operating system and processor of a node.
@@ -207,6 +241,17 @@ message GPU {
   // Zero when the driver is too old to report it, which is not the same as
   // idle: the metric is simply absent rather than wrong.
   int32 utilization_percent = 15;
+  // nic_paths is how far each network adapter is from this card, by HCA
+  // name: PIX, PXB, PHB, NODE or SYS, from nvidia-smi topo.
+  map<string, string> nic_paths = 16;
+  // The card's multi-node NVLink membership, from nvidia-smi -q: state is
+  // Completed once the fabric manager has joined it; cluster_uuid and
+  // clique_id name the domain it joined.
+  string fabric_state = 17;
+  string fabric_status = 18;
+  string cluster_uuid = 19;
+  string clique_id = 20;
+  string fabric_bandwidth = 21;
 }
 
 message FabricAttachment {
@@ -231,6 +276,150 @@ message LocalWeights {
   bool shared = 3;
 }
 
+// NodeLoad is a node's use of its own CPU and memory, and each engine's share.
+//
+// Carried on the heartbeat rather than scraped from the node: the control
+// plane already exports the fleet from one place, and a node exporter per
+// machine would be a scrape target, a port and a firewall rule each.
+message NodeLoad {
+  // cpu_percent is the whole machine, 0 to 100 however many cores it has.
+  double cpu_percent = 1;
+  int32 cpu_count = 2;
+  // The kernel's run-queue averages over 1, 5 and 15 minutes.
+  double load1 = 3;
+  double load5 = 4;
+  double load15 = 5;
+  // memory_used_bytes is total less available: what the machine could not
+  // hand to a new process without evicting something.
+  int64 memory_used_bytes = 6;
+  int64 memory_available_bytes = 7;
+  repeated ShardLoad shards = 8;
+  // What the machine's own sensors say; zero is not measured. cpu_temp_c is
+  // the hottest CPU sensor, cpu_power_w the CPU packages' draw from RAPL,
+  // inlet_temp_c the board's inlet or ambient sensor where it has one.
+  double cpu_temp_c = 9;
+  double cpu_power_w = 10;
+  double inlet_temp_c = 11;
+  repeated GPUSensor gpus = 12;
+}
+
+// GPUSensor is one card's heat, power, and what is holding it back.
+message GPUSensor {
+  int32 index = 1;
+  double temp_c = 2;
+  double memory_temp_c = 3;
+  double power_w = 4;
+  double power_limit_w = 5;
+  // throttle names what is holding the clocks down now: sw_power_cap,
+  // hw_slowdown, sw_thermal, hw_thermal, power_brake.
+  repeated string throttle = 6;
+  // How busy the card is, sampled with the rest, rather than read once at
+  // registration. -1 is not reported by the driver. memory_total_mb is set
+  // by every agent that sends these, so zero there is an older agent.
+  double utilization_percent = 7;
+  double memory_utilization_percent = 8;
+  double memory_used_mb = 9;
+  double memory_total_mb = 10;
+  double sm_clock_mhz = 11;
+  // ecc_uncorrected counts uncorrected ECC errors since the driver loaded.
+  double ecc_uncorrected = 12;
+  // What wears the card out, read every minute; has_wear says these are
+  // present. Counts of -1 are not reported for this card.
+  bool has_wear = 13;
+  int64 pcie_replays = 14;
+  int64 retired_pages_sbe = 15;
+  int64 retired_pages_dbe = 16;
+  bool retirement_pending = 17;
+  int64 remapped_rows_correctable = 18;
+  int64 remapped_rows_uncorrectable = 19;
+  bool remap_pending = 20;
+  bool remap_failure = 21;
+  int64 nvlink_crc_errors = 22;
+  int64 nvlink_replay_errors = 23;
+  int64 nvlink_recovery_errors = 24;
+  // xid_errors are the driver's error events since the agent started.
+  int64 xid_errors = 25;
+  int32 last_xid = 26;
+  string last_xid_text = 27;
+  int64 last_xid_unix = 28;
+}
+
+// ShardLoad is one engine process group's use.
+message ShardLoad {
+  string shard_id = 1;
+  // cpu_cores is in cores, the way top counts a process: 2.5 is two and a
+  // half cores busy, which a percentage of the machine would hide on a large
+  // one.
+  double cpu_cores = 2;
+  int64 memory_rss_bytes = 3;
+}
+
+// IBPort is one InfiniBand or RoCE port's state and counters.
+message IBPort {
+  string hca = 1;
+  int32 port = 2;
+  // state is ACTIVE, INIT, DOWN...; phys_state tells a cable unplugged
+  // (Polling, Disabled) from a link up with no subnet manager (LinkUp and INIT).
+  string state = 3;
+  string phys_state = 4;
+  int32 rate_gbps = 5;
+  // link_layer is InfiniBand or Ethernet (RoCE).
+  string link_layer = 6;
+  // counters are cumulative, by sysfs name; data counters in bytes.
+  map<string, uint64> counters = 7;
+
+  // Identity: the adapter's node GUID and the port's own, as the fabric
+  // knows them.
+  string node_guid = 8;
+  string port_guid = 9;
+  // The subnet manager this port sees as master, on InfiniBand. Every port on
+  // one physical fabric sees the same one, which is what tells separate
+  // fabrics apart when their names are only what somebody typed.
+  string sm_guid = 10;
+  int32 sm_lid = 11;
+  int32 sm_priority = 12;
+  string sm_state = 13;
+  // sm_error says why sm_guid is empty on an active InfiniBand port:
+  // infiniband-diags missing, or no subnet manager answering.
+  string sm_error = 14;
+  // network is, on RoCE, the IP network of the RoCE v2 address: RoCE has no
+  // subnet manager, and the network is what its fabric is.
+  string network = 15;
+  string gid_prefix = 16;
+  // subnet_managers is every subnet manager the fabric's subnet
+  // administrator lists — master and standbys — asked through one port per
+  // fabric, so it is on that port only. It is what keeps a fabric one fabric
+  // through a failover: the master changes, the list does not.
+  repeated SubnetManager subnet_managers = 17;
+  string subnet_managers_error = 18;
+}
+
+// SubnetManager is one subnet manager on a fabric.
+message SubnetManager {
+  string guid = 1;
+  int32 lid = 2;
+  int32 priority = 3;
+  // state is master, standby, discovering or not active.
+  string state = 4;
+}
+
+// IMEX is the nvidia-imex service and its channels.
+message IMEX {
+  // state is systemd's word for it, empty where it is not installed.
+  string state = 1;
+  int32 channels = 2;
+}
+
+// GPUFabricState is one GPU's NVLink domain membership.
+message GPUFabricState {
+  int32 index = 1;
+  string state = 2;
+  string status = 3;
+  string cluster_uuid = 4;
+  string clique_id = 5;
+  string bandwidth = 6;
+}
+
 message Heartbeat {
   string node_id = 1;
   // gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on
@@ -271,6 +460,38 @@ message Heartbeat {
   // than "nothing to say", as for weights.
   bool provision_changed = 10;
   repeated ProvisionStatus provision = 11;
+
+  // load is how busy the machine is, sampled over the interval since the
+  // previous heartbeat. Absent on the first one, which has nothing to take a
+  // difference from, and from an agent on a system without /proc.
+  NodeLoad load = 12;
+
+  // ib_ports is every InfiniBand or RoCE port as it stands now, re-read on
+  // each heartbeat: a port that goes down after the agent started used to go
+  // unnoticed until the agent restarted.
+  repeated IBPort ib_ports = 13;
+  // fabrics is the fabrics this node can now carry traffic on, sent only when
+  // that changed since registration or the last heartbeat that said so — a
+  // port going down takes its fabric away, and one coming back returns it.
+  bool fabrics_changed = 14;
+  repeated FabricAttachment fabrics = 15;
+
+  // gpu_fabric is each GPU's NVLink domain membership as it stands now, sent
+  // once a minute: a fabric manager restarting drops every card out of its
+  // domain, and the node has to stop being offered work that spans it.
+  repeated GPUFabricState gpu_fabric = 16;
+  IMEX imex = 17;
+  // cpu_count as in Register, for a VM given more CPUs while it runs.
+  int32 cpu_count = 18;
+
+  // gpu_foreign_memory_mb, indexed by GPU, is what processes that are not
+  // this agent's shards hold on each card, read afresh every heartbeat. What
+  // keeps a card out of placement for another tenant is decided from this
+  // rather than from registration: a card that had something else on it when
+  // the agent started is not taken forever, and one that gains a tenant later
+  // is not handed out. Empty from an agent that does not measure it, or from
+  // one that could not this time.
+  repeated int32 gpu_foreign_memory_mb = 19;
 }
 
 // Provision is the software a node should have: which engines, at which
@@ -291,6 +512,9 @@ message Provision {
 message EngineWant {
   string engine = 1;  // "llamacpp"
   string version = 2; // "b10991"
+  // variant is the build to install, when one is chosen for the node rather
+  // than left to its hardware: "cu13", "cu129". Empty is the agent's choice.
+  string variant = 3;
 }
 
 // ProvisionStatus is where one engine's installation stands on a node.
@@ -377,6 +601,13 @@ message StartShard {
   // profile_fp identifies the version of the profile, so the agent can say
   // which one a running shard was started from. The agent never interprets it.
   string profile_fp = 15;
+  // api_key is what the engine demands of every request, derived by the
+  // control plane from the replica id. The router sends it; nothing else
+  // that can reach the node's ports has it.
+  string api_key = 16;
+  // cpu_threads is how many CPU threads a shard with no GPU computes on:
+  // the cores placement reserved for it. Zero leaves it to the agent.
+  int32 cpu_threads = 17;
 }
 
 message EngineProfile {
@@ -391,6 +622,9 @@ message EngineProfile {
   map<string, string> env = 9;
   string model_path = 10;
   string served_model_name = 11;
+  // adapters are LoRA adapters loaded beside the model, in order: the index
+  // is the id a request applies one by.
+  repeated LoraAdapter adapters = 12;
 }
 
 // RemoveWeights deletes cached model files a node no longer needs.
@@ -446,3 +680,87 @@ message LogsReply {
   // error says why the lines are absent: an unknown source, say.
   string error = 4;
 }
+
+// FabricScanRequest asks a node to map the fabric one of its ports is on,
+// with ibnetdiscover: every switch, every adapter and every link between
+// them. One node per fabric is asked, every few minutes; a fabric looks the
+// same from all of its ports.
+message FabricScanRequest {
+  string request_id = 1;
+  string hca = 2;
+  int32 port = 3;
+}
+
+// FabricScanReply is ibnetdiscover's output as it printed it. Parsed by the
+// control plane, so a format the parser does not yet know is a control plane
+// upgrade rather than one on every node.
+message FabricScanReply {
+  string request_id = 1;
+  string output = 2;
+  string error = 3;
+  int64 took_ms = 4;
+}
+
+// FabricTestRequest asks a node for one side of an RDMA bandwidth test
+// between two nodes: ib_write_bw, from perftest, over one adapter. The
+// server side is asked first and answers when the test is over; the client
+// dials it. What it proves is what a replica spanning nodes needs and nothing
+// on one node can show: that these two adapters reach each other through the
+// fabric, and how fast.
+message FabricTestRequest {
+  string request_id = 1;
+  // role is "server" or "client".
+  string role = 2;
+  string hca = 3;
+  int32 port = 4;
+  // peer is the server's management address, for the client. perftest
+  // exchanges its connection details over TCP before any RDMA.
+  string peer = 5;
+  // tcp_port is that exchange's port, the same on both sides.
+  int32 tcp_port = 6;
+  int32 seconds = 7;
+}
+
+// FabricTestReply is what one side measured.
+message FabricTestReply {
+  string request_id = 1;
+  // gbps is the average bandwidth, in gigabits a second.
+  double gbps = 2;
+  // output is the end of the tool's output, for when the number is not
+  // enough.
+  string output = 3;
+  string error = 4;
+  int64 took_ms = 5;
+}
+
+// UpgradeAgent tells an agent to replace its own binary with a release and
+// restart.
+//
+// The agent fetches the release for its own platform and checks it against
+// the release's checksums before replacing anything, keeps the old binary
+// beside the new, then exits so systemd starts the new one. Engines keep
+// running throughout, as for any agent restart, and are adopted on the way
+// back. Over the stream the agent already holds, so upgrading a fleet needs
+// no ssh into it.
+message UpgradeAgent {
+  string request_id = 1;
+  // version must be a release: v0.10.4 or v0.10.4-rc.2.
+  string version = 2;
+  // releases is the base URL release files are downloaded from.
+  string releases = 3;
+}
+
+// UpgradeReply says whether the new binary is in place. Sent before the
+// agent exits; an empty error means it will be back on the new version.
+message UpgradeReply {
+  string request_id = 1;
+  string version = 2;
+  string error = 3;
+}
+
+// LoraAdapter is one adapter: the name clients ask for it by, and its source,
+// which the agent fetches as it does the model's weights.
+message LoraAdapter {
+  string name = 1;
+  string source = 2;
+}

deploy/bootstrap/onboard-node.sh

@@ -35,8 +35,9 @@ usage: $0 --node ID --addr HOST --server URL [options]
   --mgmt-if NAME    interface NCCL bootstraps over (default: the one holding
                     that address)
   --llamacpp-bin P  use this llama-server instead of the one the agent installs
-  --vllm-bin P      path to the vllm entry point (not yet installed by the agent)
-  --sglang-python P python interpreter of the SGLang virtualenv
+  --vllm-bin P      use this vllm instead of the one the agent installs on a
+                    GPU node (also set engines.vllm: "" in server.yaml)
+  --sglang-python P python interpreter of an SGLang virtualenv you installed
   --nvidia-smi PATH where nvidia-smi lives, when it is not on PATH. Containerized
                     driver installs routinely put it somewhere else.
   --ib-root DIR     the sysfs InfiniBand directory (default /sys/class/infiniband)
@@ -121,9 +122,10 @@ TMP="$(mktemp -d)"
 trap 'rm -rf "$TMP"' EXIT
 "$BINARY" ctl cert issue-client "$NODE" --dir "$PKI" --out "$TMP" >/dev/null
 
-# 2 and 3. Identity, and trusting the root so future tooling on that machine
-#    does not treat our certificates as strangers.
-say "installing the certificate and trusting the root"
+# 2 and 3. Identity. The root is trusted by the agent's configuration only,
+#    never system-wide: it signs whatever the control plane asks, and every
+#    TLS client on the machine would take it for any site.
+say "installing the certificate"
 scp -q $SSHOPTS "$PKI/ca.pem" "$TMP/$NODE.pem" "$TMP/$NODE-key.pem" "$SSH_USER@$ADDR:/tmp/"
 $SSH "set -e
   sudo install -d -m 0755 /etc/llmgw
@@ -131,8 +133,10 @@ $SSH "set -e
   sudo install -m 0644 /tmp/$NODE.pem /etc/llmgw/agent.pem
   sudo install -m 0600 /tmp/$NODE-key.pem /etc/llmgw/agent-key.pem
   sudo chown root:root /etc/llmgw/*
-  sudo install -m 0644 /tmp/ca.pem /usr/local/share/ca-certificates/llmgw-ca.crt
-  sudo update-ca-certificates >/dev/null 2>&1 || true
+  if [ -f /usr/local/share/ca-certificates/llmgw-ca.crt ]; then
+    sudo rm -f /usr/local/share/ca-certificates/llmgw-ca.crt
+    sudo update-ca-certificates --fresh >/dev/null 2>&1 || true
+  fi
   rm -f /tmp/ca.pem /tmp/$NODE.pem /tmp/$NODE-key.pem"
 
 # 4. The binary, the configuration and the unit.
@@ -142,7 +146,7 @@ $SSH "set -e
 # that is not a release (a developer's, with commits past the tag) has no such
 # file, and is copied as before.
 TAG="$("$BINARY" --version 2>/dev/null | awk '{print $NF}')"
-if printf '%s' "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
+if printf '%s' "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$'; then
   say "installing llmgw $TAG from the release"
   $SSH "set -e
     arch=\$(dpkg --print-architecture)
@@ -226,6 +230,10 @@ ExecStart=/usr/local/bin/llmgw agent --config /etc/llmgw/agent.yaml
 KillMode=process
 Restart=on-failure
 RestartSec=2s
+# RDMA pins the buffers it registers, and NCCL registers a great deal: with
+# systemd's default 8 MB lock limit, an engine that drops the capability that
+# lets root ignore it fails its first collective over InfiniBand.
+LimitMEMLOCK=infinity
 StateDirectory=llmgw-agent
 
 [Install]

deploy/examples/agent.yaml

@@ -47,26 +47,24 @@ heartbeat_interval: 5s
 reconnect_base: 1s
 reconnect_max: 30s
 
-# Fabric labelling is explicit and there is no way around it. OpenSM's default
-# subnet prefix is fe80:: on every fabric, so a matching prefix proves nothing
-# and a wrong guess produces a replica that starts, passes health checks and
-# then hangs on its first all-reduce (plan §5.2).
+# Each fabric this node's adapters are on, by name. The name is checked against
+# what the adapters report — the subnet managers every port on one fabric sees —
+# so two independent fabrics given one name are placed as two and the cluster
+# page says so; give each its own. The rate is measured, not read from here.
 #
 # A node with no fabrics is its own single-node island: it can host single-node
-# replicas and will never be given a multi-node one.
+# replicas and will never be given a multi-node one. docs/fabrics.md has more.
 fabrics:
   - name: ib-a
     hcas: [mlx5_0, mlx5_1, mlx5_2, mlx5_3]
-    rate_gbps: 200
 
 discovery:
   # Worth setting where the driver lives outside the default paths, which
   # containerized driver installs often do.
   nvidia_smi: ""
   ib_root: /sys/class/infiniband
-  # NVML is the only source of NVLink topology, which is what keeps a
-  # tensor-parallel group off the PCIe path inside a node. It needs a binary
-  # built with `-tags nvml`; otherwise this silently falls back to nvidia-smi.
+  # NVLink topology comes from `nvidia-smi topo -m` on every build; a binary
+  # built with `-tags nvml` reads it from NVML instead, when this is set.
   prefer_nvml: true
 
 # Where this node keeps model weights besides /var/lib/llmgw-weights, which is
@@ -96,10 +94,11 @@ discovery:
 engines:
   vllm:
     bin: /opt/venvs/vllm/bin/vllm
-    # Which multi-node mechanism works depends on the vLLM version, and the
-    # flags have changed more than once. Check the version you deploy.
-    launch_mode: ray      # "ray" or "mp"
-    ray_bin: /opt/venvs/vllm/bin/ray
+    # How a replica spanning nodes is launched. mp, the default, is what vLLM
+    # documents today and needs nothing but vLLM; ray is for a vLLM too old
+    # for mp's multi-node flags, and needs Ray on every node.
+    launch_mode: mp       # "mp" or "ray"
+    # ray_bin: /opt/venvs/vllm/bin/ray
   sglang:
     # SGLang is launched as a module, which is its documented entry point.
     python: /opt/venvs/sglang/bin/python

deploy/examples/server.yaml

@@ -92,6 +92,11 @@ agent_listen: ":7443"
 #   flush_interval: 200ms
 #   batch_size: 500
 #   buffer_size: 8192
+#
+#   # How long the record of each request is kept (what `ctl request` reads).
+#   # Charts, totals and budgets come from five-minute sums that are kept, so
+#   # this only bounds the table that grows by a row a request. 0 keeps all.
+#   usage_retention: 2160h   # 90 days
 
 router:
   scrape_interval: 2s

deploy/ha/README.md

@@ -85,3 +85,63 @@ auth:
 
 See [docs/install.md](../../docs/install.md) for moving an existing
 installation onto PostgreSQL without reissuing every key.
+
+## The store itself: PostgreSQL under Patroni
+
+Two control planes on one PostgreSQL have moved the single point of failure,
+not removed it. Patroni keeps PostgreSQL on two machines, one streaming to the
+other, and promotes the standby when the primary is lost. It keeps its leader
+lock in etcd, and etcd decides only with a majority — so three members, not
+two: two machines that lose sight of each other must not both decide they are
+the primary. The third member can be any machine that is up; it holds a vote,
+not data.
+
+On the reference cluster:
+
+| machine | etcd | PostgreSQL |
+|---|---|---|
+| llmgw-ctl (control plane) | member | — |
+| llmgw-pg01 | member | Patroni |
+| llmgw-ctl02 (second control plane) | member | Patroni |
+
+`llmgw server init --patroni user@host --patroni user@host` sets all of it up
+over ssh and moves the store in — see [install.md](../../docs/install.md).
+It looks before it acts at every step, so it is also how an existing cluster
+is checked: run against one that is up, it changes nothing. What it writes
+is in `etcd.default.example` (one member's `/etc/default/etcd`) and
+`patroni.yml.example` (one node's `/etc/patroni/config.yml`), for anyone
+doing it by hand; passwords are generated into `/root/llmgw-pg` on each
+database machine.
+
+**Give llmgw every member, and ask for the writable one:**
+
+```
+postgres://llmgw:…@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=disable&target_session_attrs=read-write
+```
+
+The driver connects to whichever member accepts writes, and reconnects to the
+new one after a failover. No floating address or HAProxy is needed for that.
+`llmgw server backup` reads the same address and dumps from a standby when
+there is one.
+
+Measured on the reference cluster, with the gateway serving throughout:
+
+- **Planned switchover** (`patronictl switchover`): the control plane logged
+  one failed lease renewal, reconnected to the new primary, and kept its
+  leadership; nothing else noticed.
+- **The primary's machine switched off**: the standby was promoted 24 seconds
+  later — the 30-second TTL in `patroni.yml` — and the control plane rode it
+  out without losing leadership or a replica. Started again, the old primary
+  rejoined as a standby by itself.
+
+**Watching it.** For a store of several hosts the control plane reads the
+cluster from each member's Patroni (port 8008) and publishes
+`llmgw_store_member{member,host,role,state}`, `llmgw_store_replica_lag_bytes`
+and `llmgw_store_patroni_up`; a host of the address that the cluster no longer
+lists is a member in state `missing`. `llmgw ctl status` prints it as one
+line, and `deploy/monitoring/llmgw-rules.yml` alerts on no leader, a member
+not streaming or missing, and a standby too far behind to be promoted.
+
+Not done here, and worth doing in production: a watchdog (`softdog`) so a
+primary that has lost the lock but not died cannot keep accepting writes, TLS
+on etcd and between PostgreSQL members, and more than one standby.

deploy/ha/etcd.default.example

@@ -0,0 +1,10 @@
+# llmgw: one member of the three that hold Patroni's leader lock (140, 149, 150).
+ETCD_NAME=llmgw-pg01
+ETCD_DATA_DIR=/var/lib/etcd/default
+ETCD_LISTEN_PEER_URLS=http://10.0.0.149:2380
+ETCD_LISTEN_CLIENT_URLS=http://10.0.0.149:2379,http://127.0.0.1:2379
+ETCD_INITIAL_ADVERTISE_PEER_URLS=http://10.0.0.149:2380
+ETCD_ADVERTISE_CLIENT_URLS=http://10.0.0.149:2379
+ETCD_INITIAL_CLUSTER=llmgw-ctl=http://10.0.0.140:2380,llmgw-pg01=http://10.0.0.149:2380,llmgw-ctl02=http://10.0.0.150:2380
+ETCD_INITIAL_CLUSTER_STATE=new
+ETCD_INITIAL_CLUSTER_TOKEN=llmgw-patroni

deploy/ha/patroni.yml.example

@@ -0,0 +1,68 @@
+# llmgw: the control planes' store, kept by Patroni on llmgw-pg01 and
+# llmgw-ctl02, with the leader lock in etcd on 140, 149 and 150.
+scope: llmgw
+namespace: /service/
+name: llmgw-pg01
+
+restapi:
+  listen: 10.0.0.149:8008
+  connect_address: 10.0.0.149:8008
+
+etcd3:
+  hosts: 10.0.0.140:2379,10.0.0.149:2379,10.0.0.150:2379
+
+bootstrap:
+  dcs:
+    ttl: 30
+    loop_wait: 10
+    retry_timeout: 10
+    maximum_lag_on_failover: 1048576
+    postgresql:
+      use_pg_rewind: true
+      use_slots: true
+      parameters:
+        wal_level: replica
+        hot_standby: "on"
+        max_wal_senders: 10
+        max_replication_slots: 10
+        wal_keep_size: 1GB
+        wal_log_hints: "on"
+        password_encryption: scram-sha-256
+  initdb:
+    - encoding: UTF8
+    - data-checksums
+
+postgresql:
+  listen: 127.0.0.1,10.0.0.149:5432
+  connect_address: 10.0.0.149:5432
+  data_dir: /var/lib/postgresql/16/llmgw
+  bin_dir: /usr/lib/postgresql/16/bin
+  pgpass: /var/lib/postgresql/.pgpass_patroni
+  authentication:
+    superuser:
+      username: postgres
+      password: "<from /root/llmgw-pg>"
+    replication:
+      username: replicator
+      password: "<from /root/llmgw-pg>"
+    rewind:
+      username: postgres
+      password: "<from /root/llmgw-pg>"
+  parameters:
+    unix_socket_directories: /var/run/postgresql
+  pg_hba:
+    - local all all peer
+    - host all all 127.0.0.1/32 scram-sha-256
+    - host replication replicator 10.0.0.149/32 scram-sha-256
+    - host replication replicator 10.0.0.150/32 scram-sha-256
+    - host all postgres 10.0.0.149/32 scram-sha-256
+    - host all postgres 10.0.0.150/32 scram-sha-256
+    - host llmgw llmgw 10.0.0.140/32 scram-sha-256
+    - host llmgw llmgw 10.0.0.149/32 scram-sha-256
+    - host llmgw llmgw 10.0.0.150/32 scram-sha-256
+
+tags:
+  nofailover: false
+  noloadbalance: false
+  clonefrom: false
+  nosync: false

deploy/monitoring/README.md

@@ -46,6 +46,19 @@ apt-get install prometheus prometheus-alertmanager
 install -m 0644 prometheus.yml            /etc/prometheus/prometheus.yml
 install -m 0644 llmgw-rules.yml           /etc/prometheus/rules/llmgw.yml
 install -m 0644 alertmanager.yml          /etc/prometheus/alertmanager.yml
+# Alertmanager answers on loopback only. Its API takes silences from anyone who
+# can reach it, with no login, and silencing BackupTooOld or ModelParked is how
+# something that should be noticed is not. Prometheus beside it is its only
+# client; one Alertmanager needs no cluster port.
+echo 'ARGS="--web.listen-address=127.0.0.1:9093 --cluster.listen-address="' \
+    > /etc/default/prometheus-alertmanager
+# Prometheus itself is read-only as installed: its admin and lifecycle APIs
+# stay off (no --web.enable-admin-api, no --web.enable-lifecycle). It still
+# answers any query from anyone who reaches :9090, which says which models and
+# nodes there are and how busy each is. Its readers are the control plane
+# (monitoring.prometheus) and Grafana beside it; to keep it to those, allow
+# the control plane and drop the rest, e.g. with nftables:
+#   nft add rule inet filter input tcp dport 9090 ip saddr != { 127.0.0.1, CONTROL_PLANE_IP } drop
 promtool check config /etc/prometheus/prometheus.yml
 systemctl restart prometheus prometheus-alertmanager
 
@@ -53,6 +66,8 @@ install -m 0644 grafana/datasource.yaml   /etc/grafana/provisioning/datasources/
 install -m 0644 grafana/dashboards.yaml   /etc/grafana/provisioning/dashboards/llmgw.yaml
 install -m 0644 -o grafana -g grafana grafana/llmgw-overview.json \
     /var/lib/grafana/dashboards/llmgw.json
+install -m 0644 -o grafana -g grafana grafana/llmgw-tenants.json \
+    /var/lib/grafana/dashboards/llmgw-tenants.json
 systemctl restart grafana-server
 ```
 
@@ -75,7 +90,8 @@ Alertmanager groups and holds alerts and has no receiver.
 Where an alert should go — an address, a Slack workspace, a webhook — is a
 decision with credentials attached, and a monitoring system that quietly starts
 mailing people is worse than one that is visibly not wired up. Alerts are
-visible at `:9093` and in Prometheus; add a receiver to `alertmanager.yml` to
+visible in Prometheus, and at `:9093` on the monitoring host itself (`ssh -L
+9093:127.0.0.1:9093 <host>` to open its page); add a receiver to `alertmanager.yml` to
 make them travel. `critical` already routes separately so it can be pointed
 somewhere else without touching the rest.
 
@@ -86,7 +102,7 @@ An alerting system nobody has seen fire is an assumption.
 ```
 llmgw ctl cordon <node>     # work already there keeps running
 # ~2 minutes later
-curl -s http://<host>:9093/api/v2/alerts | jq '.[].labels'
+ssh <host> curl -s http://127.0.0.1:9093/api/v2/alerts | jq '.[].labels'
 llmgw ctl uncordon <node>   # the alert resolves on its own
 ```
 

deploy/monitoring/grafana/llmgw-overview.json

@@ -487,6 +487,427 @@
         "defaults": {},
         "overrides": []
       }
+    },
+    {
+      "type": "row",
+      "title": "GPUs",
+      "id": 14,
+      "gridPos": {
+        "x": 0,
+        "y": 30,
+        "w": 24,
+        "h": 1
+      },
+      "collapsed": false,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU busy, by card",
+      "description": "Share of each sample period a kernel was running, sampled every heartbeat.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 31,
+        "w": 12,
+        "h": 7
+      },
+      "id": 15,
+      "targets": [
+        {
+          "expr": "llmgw_gpu_utilization_percent",
+          "legendFormat": "{{node}} gpu {{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU memory in use, by card",
+      "description": "Framebuffer in use. Engines reserve most of it at start.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 31,
+        "w": 12,
+        "h": 7
+      },
+      "id": 16,
+      "targets": [
+        {
+          "expr": "llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes",
+          "legendFormat": "{{node}} gpu {{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU memory bandwidth busy, by card",
+      "description": "Decode is bound by memory: high here with compute low is a card waiting on memory.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 38,
+        "w": 12,
+        "h": 7
+      },
+      "id": 17,
+      "targets": [
+        {
+          "expr": "llmgw_gpu_memory_utilization_percent",
+          "legendFormat": "{{node}} gpu {{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU temperature",
+      "description": "Throttling usually starts in the high 80s.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 38,
+        "w": 12,
+        "h": 7
+      },
+      "id": 18,
+      "targets": [
+        {
+          "expr": "llmgw_gpu_temperature_celsius",
+          "legendFormat": "{{node}} gpu {{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "celsius"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU power against its limit",
+      "description": "1 is the enforced limit; a card held there is power-capped.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 45,
+        "w": 12,
+        "h": 7
+      },
+      "id": 19,
+      "targets": [
+        {
+          "expr": "llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts",
+          "legendFormat": "{{node}} gpu {{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1.1
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU clock",
+      "description": "A clock held low under load is the card throttled; see the next panel for why.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 45,
+        "w": 12,
+        "h": 7
+      },
+      "id": 20,
+      "targets": [
+        {
+          "expr": "llmgw_gpu_sm_clock_mhz",
+          "legendFormat": "{{node}} gpu {{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "rotmhz"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "What holds GPUs back",
+      "description": "Cards throttled now, by reason, and uncorrected ECC errors since the driver loaded.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 52,
+        "w": 24,
+        "h": 7
+      },
+      "id": 21,
+      "targets": [
+        {
+          "expr": "sum by (reason) (llmgw_gpu_throttled)",
+          "legendFormat": "throttled: {{reason}}",
+          "refId": "A"
+        },
+        {
+          "expr": "sum by (node, gpu) (llmgw_gpu_ecc_errors)",
+          "legendFormat": "ECC errors {{node}} gpu {{gpu}}",
+          "refId": "B"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Output tokens/s per GPU, by model",
+      "description": "What each model gets out of each card it holds.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 59,
+        "w": 12,
+        "h": 7
+      },
+      "id": 22,
+      "targets": [
+        {
+          "expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) / count by (model) (llmgw_gpu_assigned)",
+          "legendFormat": "{{model}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Output tokens per watt-hour, by model",
+      "description": "Tokens for the energy the model's cards draw.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 59,
+        "w": 12,
+        "h": 7
+      },
+      "id": 23,
+      "targets": [
+        {
+          "expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) * 3600 / sum by (model) (llmgw_gpu_power_watts * on (node, gpu) group_left (model) llmgw_gpu_assigned)",
+          "legendFormat": "{{model}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU wear",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "id": 24,
+      "description": "What wears cards out: XID errors in the last 15 minutes, memory rows remapped, PCIe retries an hour. Anything above zero here is worth a look on that card's node page.",
+      "gridPos": {
+        "x": 0,
+        "y": 66,
+        "w": 24,
+        "h": 7
+      },
+      "targets": [
+        {
+          "expr": "increase(llmgw_gpu_xid_errors[15m]) > 0",
+          "legendFormat": "XID {{node}} gpu {{gpu}}",
+          "refId": "A"
+        },
+        {
+          "expr": "sum by (node, gpu) (llmgw_gpu_remapped_rows) > 0",
+          "legendFormat": "rows remapped {{node}} gpu {{gpu}}",
+          "refId": "B"
+        },
+        {
+          "expr": "increase(llmgw_gpu_pcie_replays[1h]) > 0",
+          "legendFormat": "PCIe retries/h {{node}} gpu {{gpu}}",
+          "refId": "C"
+        },
+        {
+          "expr": "increase(llmgw_gpu_nvlink_errors[1h]) > 0",
+          "legendFormat": "NVLink {{kind}}/h {{node}} gpu {{gpu}}",
+          "refId": "D"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
     }
   ]
 }

deploy/monitoring/grafana/llmgw-tenants.json

@@ -0,0 +1,650 @@
+{
+  "uid": "llmgw-tenants",
+  "title": "llmgw \u2014 tenants",
+  "schemaVersion": 39,
+  "version": 1,
+  "editable": true,
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "refresh": "1m",
+  "tags": [
+    "llmgw"
+  ],
+  "description": "Requests, tokens, failures, refusals and first-token time per tenant \u2014 the level router.tenant_label names \u2014 and every budget's share spent.",
+  "templating": {
+    "list": [
+      {
+        "name": "tenant",
+        "label": "Tenant",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "llmgw-prom"
+        },
+        "query": {
+          "query": "label_values(llmgw_tenant_requests_total, tenant)",
+          "refId": "tenant"
+        },
+        "definition": "label_values(llmgw_tenant_requests_total, tenant)",
+        "multi": true,
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "title": "Requests a second",
+      "type": "stat",
+      "description": "Across the tenants chosen above.",
+      "gridPos": {
+        "x": 0,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
+          "legendFormat": "",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "textMode": "auto",
+        "colorMode": "value"
+      }
+    },
+    {
+      "id": 2,
+      "title": "Failing",
+      "type": "stat",
+      "description": "Requests that failed, not counting refusals by a limit.",
+      "gridPos": {
+        "x": 6,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"error\"}[5m])) / sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
+          "legendFormat": "",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 0.02
+              },
+              {
+                "color": "red",
+                "value": 0.05
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "textMode": "auto",
+        "colorMode": "value"
+      }
+    },
+    {
+      "id": 3,
+      "title": "Refused by a limit",
+      "type": "stat",
+      "description": "Rate, token and budget limits; 429 to the caller.",
+      "gridPos": {
+        "x": 12,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[5m])) / sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
+          "legendFormat": "",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 0.02
+              },
+              {
+                "color": "red",
+                "value": 0.1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "textMode": "auto",
+        "colorMode": "value"
+      }
+    },
+    {
+      "id": 4,
+      "title": "First token, p95",
+      "type": "stat",
+      "description": "What people wait for before anything appears.",
+      "gridPos": {
+        "x": 18,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(llmgw_tenant_ttft_seconds_bucket{tenant=~\"$tenant\"}[5m])))",
+          "legendFormat": "",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 3
+              },
+              {
+                "color": "red",
+                "value": 10
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "textMode": "auto",
+        "colorMode": "value"
+      }
+    },
+    {
+      "id": 5,
+      "title": "Requests by tenant",
+      "type": "timeseries",
+      "description": "Stacked, so the top edge is the whole of it.",
+      "gridPos": {
+        "x": 0,
+        "y": 4,
+        "w": 12,
+        "h": 9
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
+          "legendFormat": "{{tenant}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "custom": {
+            "stacking": {
+              "mode": "normal"
+            },
+            "fillOpacity": 30
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 6,
+      "title": "Tokens by tenant",
+      "type": "timeseries",
+      "description": "Prompt and completion together: what was consumed.",
+      "gridPos": {
+        "x": 12,
+        "y": 4,
+        "w": 12,
+        "h": 9
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (tenant) (rate(llmgw_tenant_tokens_total{tenant=~\"$tenant\"}[5m]))",
+          "legendFormat": "{{tenant}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "stacking": {
+              "mode": "normal"
+            },
+            "fillOpacity": 30
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 7,
+      "title": "Refused by a limit, by tenant",
+      "type": "timeseries",
+      "description": "The share of each tenant's requests a limit turned away.",
+      "gridPos": {
+        "x": 0,
+        "y": 13,
+        "w": 12,
+        "h": 8
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[5m])) / sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
+          "legendFormat": "{{tenant}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 8,
+      "title": "Failing, by tenant",
+      "type": "timeseries",
+      "description": "One tenant alone failing is usually what it sends; all of them together is a model.",
+      "gridPos": {
+        "x": 12,
+        "y": 13,
+        "w": 12,
+        "h": 8
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"error\"}[5m])) / sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
+          "legendFormat": "{{tenant}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 9,
+      "title": "First token p95, by tenant",
+      "type": "timeseries",
+      "description": "",
+      "gridPos": {
+        "x": 0,
+        "y": 21,
+        "w": 12,
+        "h": 8
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "histogram_quantile(0.95, sum by (tenant, le) (rate(llmgw_tenant_ttft_seconds_bucket{tenant=~\"$tenant\"}[10m])))",
+          "legendFormat": "{{tenant}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 10,
+      "title": "Budgets spent this period",
+      "type": "bargauge",
+      "description": "Every organization, team, project and person with a budget. Requests are refused at 100%.",
+      "gridPos": {
+        "x": 12,
+        "y": 21,
+        "w": 12,
+        "h": 8
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sort_desc(llmgw_budget_spent_ratio)",
+          "legendFormat": "{{scope}} {{name}} ({{period}})",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 0.8
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "displayMode": "gradient",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "showUnfilled": true
+      }
+    },
+    {
+      "id": 11,
+      "title": "Who used most, last 24 hours",
+      "type": "table",
+      "description": "Spend in money is on the people page's usage tab, which reads it from the store.",
+      "gridPos": {
+        "x": 0,
+        "y": 29,
+        "w": 24,
+        "h": 8
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sort_desc(sum by (tenant) (increase(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[24h])))",
+          "legendFormat": "requests",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          },
+          "format": "table",
+          "instant": true
+        },
+        {
+          "refId": "B",
+          "expr": "sum by (tenant) (increase(llmgw_tenant_tokens_total{tenant=~\"$tenant\"}[24h]))",
+          "legendFormat": "tokens",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          },
+          "format": "table",
+          "instant": true
+        },
+        {
+          "refId": "C",
+          "expr": "sum by (tenant) (increase(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[24h]))",
+          "legendFormat": "limited",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          },
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "options": {
+        "showHeader": true
+      },
+      "transformations": [
+        {
+          "id": "merge",
+          "options": {}
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true
+            },
+            "renameByName": {
+              "Value #A": "Requests",
+              "Value #B": "Tokens",
+              "Value #C": "Refused by a limit"
+            }
+          }
+        }
+      ]
+    }
+  ]
+}

deploy/monitoring/llmgw-rules.yml

@@ -42,10 +42,119 @@ groups:
         annotations:
           summary: "{{ $labels.model }} has no replica ready"
           description: >-
-            The model is placed and nothing is serving it. Ten minutes, because
-            a large model on CPU legitimately takes minutes to load. A model
-            that is deliberately stopped publishes no series at all, so this
-            cannot fire for one nobody asked for.
+            The model should be running and nothing is serving it. Ten minutes,
+            because a large model on CPU legitimately takes minutes to load. The
+            series is published at 0 for every model that should be running —
+            before v0.10.4 it was published only while a replica was ready, so
+            this could not fire at all. A model deliberately stopped, or an
+            on-demand one nobody is asking for, publishes nothing.
+
+      - alert: ModelParked
+        expr: llmgw_model_parked == 1
+        labels: { severity: critical }
+        annotations:
+          summary: "{{ $labels.model }} failed every time it started and is no longer retried"
+          description: >-
+            Five replicas in a row failed at start. Before this existed a model
+            whose weights its engine could not load was started 155 times over
+            eight hours. `llmgw ctl status` has the last error; once it is dealt
+            with, `llmgw ctl retry <model>`. Changing the model's definition
+            also retries it.
+
+      - alert: ModelFailingToStart
+        expr: llmgw_model_start_failures >= 2
+        for: 5m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.model }} has failed to start {{ $value }} times in a row"
+          description: >-
+            On its way to being parked. The engine's own words are in
+            `llmgw ctl logs <node> --source <shard>` for the next attempt, or in
+            the control plane's log for the last one.
+
+      - alert: ModelCannotBePlaced
+        expr: llmgw_model_unplaceable == 1
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.model }} is wanted and fits on no node"
+          description: >-
+            Short of replicas, and the last attempt found nowhere with the
+            memory, cards or engine it needs. `llmgw ctl explain-placement
+            <model>` says what each node lacks. For an on-demand model, requests
+            for it are told this rather than asked to retry.
+
+      - alert: ModelPreempted
+        expr: increase(llmgw_model_preempted_total[15m]) > 0
+        labels: { severity: info }
+        annotations:
+          summary: "{{ $labels.model }} ({{ $labels.class }}) gave up replicas to a higher class"
+          description: >-
+            Capacity ran short and a model of a higher class took this one's
+            room. Expected now and then; if it keeps firing for the same model,
+            the cluster is too small for what is asked of it at these classes.
+            `llmgw ctl models` shows who it gave way to.
+
+      - alert: CapacityAtLimit
+        expr: llmgw_capacity_holds >= on(scope, resource) llmgw_capacity_limit
+        for: 15m
+        labels: { severity: info }
+        annotations:
+          summary: "{{ $labels.scope }} holds all the {{ $labels.resource }} it may"
+          description: >-
+            Its models get no further replicas, under load or not, until
+            something of theirs stops or the limit is raised: `llmgw ctl quota
+            list`, `llmgw ctl quota set`.
+
+      - alert: EngineInstallFailed
+        expr: llmgw_node_engine_install{state="failed"} == 1
+        for: 15m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.engine }} {{ $labels.version }} will not install on {{ $labels.node }}"
+          description: >-
+            The node takes no model needing it until it does. The agent retries
+            on its own; `llmgw ctl nodes` and the node's logs on the cluster
+            page say why it failed.
+
+      - alert: ControlPlaneLeadership
+        expr: sum(llmgw_control_plane_leader) != 1
+        for: 2m
+        labels: { severity: critical }
+        annotations:
+          summary: "{{ $value }} control planes believe they are placing work"
+          description: >-
+            None means nothing is placed or replaced; two means both are, each
+            into GPUs the other believes free. The lease is in the store: check
+            it is reachable from every control plane.
+
+      - alert: CertificateExpiresSoon
+        expr: (llmgw_certificate_expiry_timestamp_seconds - time()) < 30 * 86400
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.cert }} ({{ $labels.subject }}) expires in {{ $value | humanizeDuration }}"
+          description: >-
+            Certificates last a year and nothing renews them. When the agent
+            server's runs out every agent refuses to connect at once. Reissue
+            with `llmgw ctl cert` and restart the control plane.
+
+      - alert: CertificateExpiresVerySoon
+        expr: (llmgw_certificate_expiry_timestamp_seconds - time()) < 7 * 86400
+        labels: { severity: critical }
+        annotations:
+          summary: "{{ $labels.cert }} ({{ $labels.subject }}) expires in {{ $value | humanizeDuration }}"
+          description: See CertificateExpiresSoon; a week is not long.
+
+      - alert: BackupTooOld
+        expr: time() - llmgw_backup_last_success_timestamp_seconds > 26 * 3600
+        labels: { severity: warning }
+        annotations:
+          summary: "The newest llmgw backup is {{ $value | humanizeDuration }} old"
+          description: >-
+            llmgw-backup.timer runs nightly. Also fires when there has never
+            been a backup (the series is 0 then). `journalctl -u
+            llmgw-backup.service` on the control plane says why the last one
+            failed; docs/install.md, "Backing up", says how to set it up.
 
       - alert: PoolSaturated
         expr: llmgw_pool_saturation > 1
@@ -91,6 +200,94 @@ groups:
             is skipped by placement rather than failing, so this is quiet until
             somebody wonders why a model will not land there.
 
+  - name: llmgw-gpus
+    rules:
+      - alert: GPUThermalThrottling
+        expr: llmgw_gpu_throttled{reason=~"sw_thermal|hw_thermal|hw_slowdown"} == 1
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }} is slowed by heat ({{ $labels.reason }})"
+          description: >-
+            Ten minutes held back by temperature: every replica on the card is
+            slower than its measurements say. Usually airflow, a fan, or the
+            room — the node page shows the card's temperature over time.
+
+      - alert: GPUUncorrectedECCErrors
+        expr: llmgw_gpu_ecc_errors > 0
+        for: 1m
+        labels: { severity: critical }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }} has {{ $value }} uncorrected ECC errors"
+          description: >-
+            Uncorrected memory errors since the driver loaded: what runs on the
+            card may be computing on corrupted memory. Drain the node
+            (llmgw ctl drain) and have the card looked at.
+
+      - alert: GPUXidError
+        expr: increase(llmgw_gpu_xid_errors[15m]) > 0
+        for: 0m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }} logged an XID error"
+          description: >-
+            The driver reported an error on the card (its code is
+            llmgw_gpu_last_xid, and the node page says it in words). 79 is a
+            card fallen off the bus, 48 and 63/64 memory errors, 74 NVLink,
+            94/95 contained and uncontained ECC errors: those need the card
+            looked at. Others are often an engine's own fault, and pass.
+
+      - alert: GPUOutOfSpareRows
+        expr: llmgw_gpu_remap_failure == 1
+        for: 1m
+        labels: { severity: critical }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }} has no spare memory rows left"
+          description: >-
+            The card cannot remap another failing row. Drain the node and
+            have the card replaced.
+
+      - alert: GPUNeedsReset
+        expr: llmgw_gpu_remap_pending == 1 or llmgw_gpu_retirement_pending == 1
+        for: 5m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }} has memory repairs waiting for a reset"
+          description: >-
+            Rows remapped or pages retired take effect only after the card is
+            reset. Drain the node (llmgw ctl drain), reset or reboot, and bring
+            it back.
+
+      - alert: GPUPCIeRetriesClimbing
+        expr: increase(llmgw_gpu_pcie_replays[1h]) > 100
+        for: 0m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }}: {{ $value }} PCIe retries in an hour"
+          description: >-
+            The card's link to the host is retrying. A riser, the slot or a
+            cable is the usual cause, and transfers to the card slow down.
+
+      - alert: GPUNVLinkErrorsClimbing
+        expr: increase(llmgw_gpu_nvlink_errors{kind="crc"}[1h]) > 10
+        for: 0m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }}: NVLink CRC errors climbing"
+          description: >-
+            Errors on the card's NVLink links: tensor-parallel replicas on it
+            slow down, and a link that keeps failing takes a replica with it.
+
+      - alert: GPUHot
+        expr: llmgw_gpu_temperature_celsius > 87
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }} at {{ $value }} °C"
+          description: >-
+            Ten minutes above 87 °C, where most data-centre cards begin to
+            throttle.
+
   - name: monitoring-host
     rules:
       - alert: MonitoringDiskFilling
@@ -153,3 +350,130 @@ groups:
             Working as designed when the cluster is full: the lowest band is
             held first. Worth knowing about because the callers being refused
             do not know why.
+
+  # Per tenant: the level router.tenant_label names (a team by default), and
+  # every organization, team, project and person given a budget.
+  - name: llmgw-tenants
+    rules:
+      - alert: BudgetMostlySpent
+        expr: llmgw_budget_spent_ratio >= 0.8 and llmgw_budget_spent_ratio < 1
+        for: 5m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.scope }} {{ $labels.name }} has spent {{ $value | humanizePercentage }} of its {{ $labels.period }}ly budget"
+          description: >-
+            Its requests are refused once it reaches all of it, until the
+            period turns. Raise it on the people page, or tell them now
+            rather than when they are refused.
+
+      - alert: BudgetSpent
+        expr: llmgw_budget_spent_ratio >= 1
+        for: 1m
+        labels: { severity: critical }
+        annotations:
+          summary: "{{ $labels.scope }} {{ $labels.name }} has spent its {{ $labels.period }}ly budget"
+          description: >-
+            Every request under it is refused with 429 until the period
+            turns, unless a cheaper model is named as its fallback.
+
+      - alert: TenantHittingLimits
+        expr: >-
+          sum by (tenant) (rate(llmgw_tenant_requests_total{outcome="limited"}[10m]))
+          / sum by (tenant) (rate(llmgw_tenant_requests_total[10m])) > 0.1
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $value | humanizePercentage }} of {{ $labels.tenant }}'s requests are refused by a limit"
+          description: >-
+            A rate, token or budget limit is turning their traffic away. Either
+            the limit is too low for what they now do, or something of theirs
+            is looping.
+
+      - alert: TenantErrors
+        expr: >-
+          sum by (tenant) (rate(llmgw_tenant_requests_total{outcome="error"}[10m]))
+          / sum by (tenant) (rate(llmgw_tenant_requests_total[10m])) > 0.05
+          and sum by (tenant) (rate(llmgw_tenant_requests_total[10m])) > 0.05
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $value | humanizePercentage }} of {{ $labels.tenant }}'s requests fail"
+          description: >-
+            Not counting refusals by a limit. If every tenant fails alike it
+            is a model (ModelFailing); one alone is usually what they send —
+            prompts past the context, a model they may not use.
+
+      - alert: TenantTrafficSpike
+        expr: >-
+          sum by (tenant) (rate(llmgw_tenant_requests_total[30m]))
+          > 3 * sum by (tenant) (rate(llmgw_tenant_requests_total[7d]))
+          and sum by (tenant) (rate(llmgw_tenant_requests_total[30m])) > 0.2
+        for: 15m
+        labels: { severity: info }
+        annotations:
+          summary: "{{ $labels.tenant }} is sending three times its weekly average"
+          description: >-
+            A new workload, a batch job, or a client retrying in a loop. Only
+            for a tenant sending more than one request every five seconds, so
+            a quiet team starting work in the morning does not page anybody.
+
+      - alert: TenantSlowFirstToken
+        expr: >-
+          histogram_quantile(0.95,
+            sum by (tenant, le) (rate(llmgw_tenant_ttft_seconds_bucket[15m]))) > 10
+          and sum by (tenant) (rate(llmgw_tenant_ttft_seconds_count[15m])) > 0.02
+        for: 15m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.tenant }} waits {{ $value | humanizeDuration }} for a first token at p95"
+          description: >-
+            What their people feel. Compare with SlowFirstToken per model: if
+            only this tenant is slow, it is the models it uses or its queue
+            band, not the cluster.
+  # The store, when it is a replicated cluster under Patroni (llmgw server
+  # init --patroni). Published only for a store of several hosts.
+  - name: llmgw-store
+    rules:
+      - alert: StoreHasNoLeader
+        expr: llmgw_store_patroni_up == 1 and on() count(llmgw_store_member{role="leader"}) == 0
+        for: 1m
+        labels: { severity: critical }
+        annotations:
+          summary: "The store's cluster has no primary"
+          description: >-
+            Every write fails — keys, usage, budgets, leadership. Patroni
+            promotes a standby within its 30-second TTL; a minute without one
+            means it cannot: etcd has lost its majority, or no standby is fit.
+            `patronictl -c /etc/patroni/config.yml list` on a member says which.
+
+      - alert: StorePatroniUnreachable
+        expr: llmgw_store_patroni_up == 0
+        for: 2m
+        labels: { severity: warning }
+        annotations:
+          summary: "No member of the store's cluster answers on its Patroni port"
+          description: >-
+            The control plane reads the cluster's state from port 8008 on each
+            member. The database may still be serving; nothing here can tell.
+
+      - alert: StoreReplicaNotStreaming
+        expr: llmgw_store_member{role!="leader", state!="streaming"} == 1
+        for: 5m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.member }} is {{ $labels.state }}, not streaming from the primary"
+          description: >-
+            A standby that is not streaming is not a standby: if the primary
+            is lost now, there is nothing to promote. "missing" is a host of
+            the store address the cluster no longer lists — usually a machine
+            that is down.
+
+      - alert: StoreReplicaBehind
+        expr: llmgw_store_replica_lag_bytes > 64 * 1024 * 1024 or llmgw_store_replica_lag_bytes == -1
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.member }} is {{ $value | humanize1024 }}B behind the primary"
+          description: >-
+            Patroni will not promote a standby more than 1 MB behind, so a
+            failover now would find none fit. -1 is a lag Patroni cannot read.

deploy/proxy/README.md

@@ -10,6 +10,9 @@ put it behind a proxy that holds a publicly trusted certificate.
 
 Both are also attached to every release.
 
+The `/about` page shows both, from copies in `internal/web/proxy`; a test fails
+when those differ from these, so change them here and copy them over.
+
 Four things both files get right, and any other proxy has to as well:
 
 - **Streaming is not buffered.** Replies are server-sent events; a buffering

deploy/systemd/llmgw-agent.service

@@ -11,6 +11,10 @@ ExecStart=/usr/local/bin/llmgw agent --config /etc/llmgw/agent.yaml
 KillMode=process
 Restart=on-failure
 RestartSec=2s
+# RDMA pins the buffers it registers, and NCCL registers a great deal: with
+# systemd's default 8 MB lock limit, an engine that drops the capability that
+# lets root ignore it fails its first collective over InfiniBand.
+LimitMEMLOCK=infinity
 StateDirectory=llmgw-agent
 
 [Install]

deploy/systemd/llmgw-backup.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=llmgw backup: the store and /etc/llmgw, to /var/backups/llmgw-server
+After=llmgw-server.service postgresql.service
+
+[Service]
+Type=oneshot
+# Nothing it writes is for anyone else: the archive holds the CA's key and a
+# copy of the store.
+UMask=0077
+# Root, because the archive includes /etc/llmgw/secrets and the certificate
+# authority's key, which the llmgw user deliberately cannot read all of.
+ExecStart=/usr/local/bin/llmgw server backup --config /etc/llmgw/server.yaml --keep 14
+Nice=10
+IOSchedulingClass=idle

deploy/systemd/llmgw-backup.timer

@@ -0,0 +1,12 @@
+[Unit]
+Description=Back up llmgw every night
+
+[Timer]
+OnCalendar=*-*-* 03:15:00
+# A machine that was off at 03:15 backs up when it next starts, rather than
+# waiting another day with a gap nobody notices until the alert.
+Persistent=true
+RandomizedDelaySec=10m
+
+[Install]
+WantedBy=timers.target

docs/fabrics.md

@@ -0,0 +1,210 @@
+# InfiniBand fabrics and NVLink
+
+For a site whose GPU nodes are joined by InfiniBand or RoCE, by NVLink, or both
+— several independent fabrics, NVL72 racks, eight-GPU HGX boards. It says what
+llmgw does with them, what it needs from you before it can, how to check a node
+in one command, and where to look when a replica spanning nodes will not run.
+
+Read [install.md](install.md) first; this is the part of it that only matters
+once there is a fabric.
+
+> **Where this stands.** Every piece here was written against the real formats
+> of `sminfo`, `saquery`, `ibnetdiscover`, `nvidia-smi topo -m` and
+> `nvidia-smi -q`, and tested against them — but none of it has yet run on a
+> site with the hardware. Run `llmgw doctor --json` on two or three nodes first
+> and keep the reports: if a parser meets a format it does not know, that report
+> is what fixes it, and the fix is a control plane upgrade, not one per node.
+
+## What llmgw does with them
+
+**Two kinds of interconnect.** InfiniBand or RoCE joins nodes (scale-out).
+NVLink joins GPUs — inside one machine on an HGX board, or across the nodes of a
+rack on an NVL72 or an NVLink Switch system (scale-up). A replica spanning nodes
+is placed, in order of preference:
+
+1. inside one **multi-node NVLink domain**, if any has enough usable nodes:
+   tensor parallelism between its nodes then runs over NVLink;
+2. on one **physical InfiniBand or RoCE fabric**, preferring nodes under one
+   leaf switch and putting nodes with a degraded cable last.
+
+It is never split across two fabrics or two NVLink domains. Within a node, the
+GPUs a shard gets are the set with the most NVLink pairs, then the nearest to
+the fabric's adapters, then on one NUMA node.
+
+**A fabric has a name and an identity.** The name is what `agent.yaml` says; the
+identity is what the adapters report — on InfiniBand the subnet managers every
+port on one fabric sees, on RoCE the IP network of its RoCE v2 address. The name
+is how you address a fabric; the identity is how llmgw catches the name being
+wrong: two independent fabrics given one name are placed as two, and the
+cluster page says so.
+
+## Before a node joins
+
+On every GPU node:
+
+| What | Why | Check |
+|---|---|---|
+| NVIDIA driver | everything | `nvidia-smi` |
+| `nvidia-fabricmanager`, on an NVSwitch machine | without it CUDA does not start on those GPUs | `systemctl is-active nvidia-fabricmanager` |
+| `rdma-core`, `ib_uverbs` loaded | user-space RDMA at all | `ls /sys/class/infiniband`, `lsmod \| grep ib_uverbs` |
+| `nvidia_peermem` loaded | GPUDirect RDMA; without it NCCL copies every transfer through host memory | `lsmod \| grep nvidia_peermem` |
+| `infiniband-diags` | tells llmgw which fabric a port is on; the agent installs it if missing | `which sminfo` |
+
+On an NVL72 or NVLink Switch system, also:
+
+| What | Why | Check |
+|---|---|---|
+| `nvidia-imex` running, every node of the domain in `/etc/nvidia-imex/nodes_config.cfg` | GPUs in different nodes share memory through it | `systemctl is-active nvidia-imex` |
+| an IMEX channel (`options nvidia NVreg_CreateImexChannel0=1`) | an engine opens one | `ls /dev/nvidia-caps-imex-channels` |
+| every GPU joined to its domain | the fabric manager does this | `nvidia-smi -q \| grep -A6 Fabric` shows `State: Completed` |
+
+On each InfiniBand fabric:
+
+- **A subnet manager, and a standby.** Without one, ports come up to `INIT` and
+  stay there; without a standby, the fabric keeps its routes if the master
+  stops but cannot recover from any change after. llmgw watches this but does
+  not run one.
+- **Its own name** in `agent.yaml` on every node attached to it. Optionally, a
+  distinct `subnet_prefix` in each subnet manager's configuration, which makes
+  GIDs unique across fabrics; llmgw does not need it.
+
+## Declaring the fabrics
+
+In `/etc/llmgw/agent.yaml`, the adapters on each fabric, by name:
+
+```yaml
+fabrics:
+  - name: ib-compute
+    hcas: [mlx5_0, mlx5_1, mlx5_2, mlx5_3, mlx5_6, mlx5_7, mlx5_8, mlx5_9]
+  - name: ib-storage
+    hcas: [mlx5_4]
+```
+
+or `--fabric ib-compute:mlx5_0,mlx5_1,…` when onboarding ([node-onboarding.md](node-onboarding.md)).
+An adapter belongs to one fabric; a node may be on several. A replica on
+`ib-compute` is given only `ib-compute`'s adapters (`NCCL_IB_HCA`).
+
+The rate is measured, not taken from the file: a fabric's rate on a node is the
+slowest of its active adapters, because a collective runs at the speed of its
+slowest link.
+
+The agent checks the declaration against the adapters on every heartbeat. An
+adapter that is missing, or whose port is down, takes that fabric off the node
+within seconds — the node stops being offered multi-node work on it — and
+returns it when the port comes back.
+
+## Checking a node: `llmgw doctor`
+
+```
+sudo llmgw doctor                 # uses /etc/llmgw/agent.yaml when present
+sudo llmgw doctor --json > $(hostname)-doctor.json
+```
+
+It reads the node and says what is wrong and how to fix it, most serious first,
+and exits non-zero when anything fails:
+
+```
+FAIL  nvlink  this machine has NVSwitch but nvidia-fabricmanager is inactive: CUDA will not initialise on its GPUs
+              → systemctl enable --now nvidia-fabricmanager (its version must match the driver's)
+WARN  fabric  nvidia_peermem is not loaded: GPUDirect RDMA is off, and NCCL copies every transfer through host memory
+              → modprobe nvidia_peermem
+OK    gpu     8 GPUs: 8× NVIDIA H100 80GB HBM3
+OK    nvlink  NVLink groups: [0 1 2 3 4 5 6 7]
+OK    fabric  mlx5_0/1 ACTIVE at 400 Gb/s (InfiniBand), subnet manager 0x248a07030030f0c0 (LID 1, SMINFO_MASTER)
+OK    fabric  subnet managers of 0x248a07030030f0c0's fabric: 0x248a07030030f0c0 (master, priority 15), 0x248a07030030f1d4 (standby, priority 14)
+OK    agent   fabric ib-compute: mlx5_0,mlx5_1,… usable at 400 Gb/s
+OK    agent   NCCL bootstraps over ens3 (10.1.0.21)
+```
+
+followed by each GPU's NVLink peers and nearest adapter, each port's state,
+rate and errors, and the NCCL environment a shard on each fabric would be given.
+`--json` writes every fact it read as well as what it concluded — send that
+file when the report is not enough.
+
+## Checking two nodes: `llmgw ctl fabric-test`
+
+Every check on one node can pass while two nodes still cannot reach each other
+— a cable into the wrong switch, a partition key, a RoCE network with no route
+— and a replica spanning them then hangs on its first all-reduce. Before
+placing one, on the control plane:
+
+```
+llmgw ctl fabric-test gpu01 gpu02
+```
+
+```
+gpu01 → gpu02 on ib-compute, 41s
+
+RAIL              LINK      MEASURED    VERDICT  DETAIL
+mlx5_0 → mlx5_0   400 Gb/s  388.2 Gb/s  ok       97% of 400 Gb/s
+mlx5_1 → mlx5_1   400 Gb/s  241.0 Gb/s  slow     60% of 400 Gb/s: check the cable, and the card's PCIe width and NUMA node
+mlx5_2 → mlx5_2   400 Gb/s  0.0 Gb/s    fail     gpu01: ib_write_bw: Couldn't connect to 10.1.0.22:18517
+```
+
+It runs `ib_write_bw` (perftest, installed on a node the first time) from each
+of the first node's adapters on the fabric to the same adapter on the second —
+mlx5_0 to mlx5_0, the way NCCL pairs them — one rail at a time, through the
+agents, so no SSH between nodes is needed. Each rail is graded against its
+link's rate: **ok** at 80% or more, **slow** at 50%, **fail** below that or when
+nothing got through. `--fabric` picks one when the nodes share several,
+`--seconds` the length of each rail's run. The traffic is real: run it when the
+fabric is quiet.
+
+It measures RDMA between adapters, not GPUDirect or NCCL itself: a pass says
+the fabric carries traffic between these two nodes at its rate, which is what
+a first all-reduce needs and what nothing on one node can show.
+
+## Watching them
+
+- **Cluster page, Fabrics.** Each declared fabric and what it physically is;
+  a table of subnet managers per fabric — master, priority, standbys, the last
+  change of master — and every disagreement: one name on two fabrics, two names
+  on one, an adapter up but undeclared, ports up with no subnet manager, a
+  fabric with no standby, a recent failover.
+- **`/admin/fabrics`.** Each fabric's map, from `ibnetdiscover` run by one of
+  its nodes every five minutes: switches in their tiers, adapters under their
+  leaves, cables coloured by rate, degraded ones in red, rail alignment, and
+  the cluster's own adapters live with their traffic. Click anything for its
+  detail.
+- **`/admin/nvlink`.** Each multi-node NVLink domain as a rack of trays: which
+  GPUs are joined, which node's IMEX is down, which link runs degraded.
+- **A node's page.** Its ports with state, rate, errors and what they are
+  physically on; its GPU-by-GPU NVLink matrix; its load.
+- **Metrics**, for alerts:
+
+| Metric | Alert when |
+|---|---|
+| `llmgw_ib_port_active` | a declared port is 0 |
+| `llmgw_ib_port_counter_total{counter="symbol_error"}` and `link_downed` | rising |
+| `llmgw_ib_port_rate_gbps` | below its usual rate |
+| `llmgw_fabric_subnet_managers{state="standby"}` | absent or 0 |
+| `llmgw_fabric_master_sm` | its `sm_guid` changes (a failover) |
+| `llmgw_fabric_problems{level="fail"}` | above 0 |
+| `llmgw_gpu_nvlink_joined` | 0 |
+| `llmgw_imex_up` | 0 |
+
+## What an engine is given
+
+For a shard on an InfiniBand or RoCE fabric:
+
+| Variable | Value |
+|---|---|
+| `NCCL_IB_HCA` | that fabric's adapters on this node, and no others |
+| `NCCL_IB_DISABLE` | `0` (`1` for a single-node shard) |
+| `NCCL_SOCKET_IFNAME`, `GLOO_SOCKET_IFNAME` | the node's management interface: `mgmt_interface`, or the one holding its management address |
+| `NCCL_IB_GID_INDEX` | on RoCE only: the RoCE v2 entry with an IPv4 address |
+| `NCCL_MNNVL_ENABLE`, `NCCL_CUMEM_ENABLE` | `1`, for a replica inside a multi-node NVLink domain |
+
+A profile's `env` overrides any of them. vLLM launches a multi-node replica
+with its own multiprocessing (`engines.vllm.launch_mode: mp`, the default):
+every node runs `vllm serve` with its rank and the leader's address, the
+workers headless. `launch_mode: ray` is there for a vLLM too old for that.
+
+## When a replica spanning nodes will not run
+
+| It | Look at |
+|---|---|
+| is not placed | `llmgw ctl explain-placement <model>`: a fabric split by identity, a node whose IMEX is down, a node refused because its adapters are cabled to two fabrics |
+| starts and hangs on its first collective | `llmgw ctl fabric-test` between two of its nodes; the cluster page's fabric problems; `doctor` on each node — a port down, the wrong interface, RoCE without a v2 GID |
+| runs, slowly | `nvidia_peermem`; the map for a degraded cable; the node page's NVLink matrix for a group that crossed PCIe |
+| worked yesterday | the subnet manager table for a failover; `link_downed` on the node's ports |

docs/facility-aware-placement.md

@@ -0,0 +1,303 @@
+# Facility-aware placement: rack power, temperature and cooling
+
+**Status: step 0, reading the nodes' own sensors, is built; steps 1–5 are
+deferred** until the rack, PDU and power information of a real site is at
+hand — without it they would be designed against guesses.
+This is the design for step 5 of resource allocation, after reserved CPU cores
+and memory (1), capacity classes and preemption (2), tenant capacity quotas (3)
+and sizing by measured throughput (4), all of which shipped in v0.10.5. Of
+what follows, only the node sensors (rollout step 0) exist in the code: they
+are read, published and shown, and change no decision.
+
+## Why
+
+In a large installation the scarce thing is often not a GPU but the room it
+sits in. Three failures are invisible to placement today:
+
+- **A rack goes over its power budget.** Every node in it reports healthy;
+  placement sees free cards and fills them; the breaker, or the facility's
+  power cap, decides the rest. A rack with A and B feeds is worse: it runs
+  fine until one feed fails and the other cannot carry the load.
+- **Hot hardware is slow hardware.** GPUs throttle long before they fail. A
+  replica placed in a rack whose inlet air is too warm serves fewer tokens per
+  second than its measured capacity says, and nothing connects the two.
+- **Correlated failure.** Two replicas of a model on two nodes in the same rack
+  or on the same PDU are one failure away from zero. Placement spreads replicas
+  across nodes, not across racks.
+
+The goal is for placement to treat facility limits the way it already treats
+memory and cores: reserve against a budget, refuse what does not fit, say why,
+and never let missing data stop the cluster.
+
+## What it builds on
+
+| Already in place | Used for |
+|---|---|
+| Node labels, reported by agents, matched by `node_selector` | Where a node is: its rack, row, PDU |
+| `inventory.Allocation` carries GPUs, memory and cores per shard | Adding watts, summed per rack and PDU |
+| Placement rejections (`explain-placement`, headroom's "why not") | Naming the rack that is full or hot |
+| Headroom: the scheduler dry-run on a copy of the inventory | Room that respects power budgets, for free |
+| Preemption with classes, minimums and a dry run (step 2) | Shedding load from a hot rack in the right order |
+| Measured throughput per profile (step 4) | Tokens per watt as a placement preference |
+| Node sensors, read by every agent (step 0, built) | Heat and power with no facility integration: see below |
+| `internal/promq`, a Prometheus client | Reading facility metrics that DCIM exporters publish |
+
+## Concepts
+
+**Location.** Each node has a place in a tree: site, room, row, rack. Power has
+its own tree beside it: a rack draws from one or more PDUs, and each PDU from a
+feed. Both are read from node labels with well-known keys:
+
+```yaml
+# agent.yaml
+labels:
+  llmgw.io/site: sto1
+  llmgw.io/room: hall-b
+  llmgw.io/row: "12"
+  llmgw.io/rack: r12-07
+  llmgw.io/pdu: r12-07-a,r12-07-b   # both feeds, for a dual-fed rack
+```
+
+A node without these labels is its own rack of one: placement behaves as it
+does today.
+
+**Domain.** A rack, a row, a PDU, a room: anything with a budget or a
+condition. Domains are declared in a facility file (below) or discovered from
+node labels; a label naming a domain the file does not declare creates it with
+no budget and no condition.
+
+**Budget.** A domain may have a power cap in watts. Placement reserves each
+replica's estimated draw against every domain its node belongs to and refuses a
+placement that would take any of them past its cap less a safety margin. For a
+dual-fed rack the cap that matters is what one feed can carry alone, since that
+is what the rack must survive.
+
+**Condition.** A domain has a thermal and cooling condition: `ok`, `warm`, `hot`
+or `unknown`, derived from readings against thresholds with hysteresis.
+Conditions change what placement prefers and allows; they are not amounts to
+fill.
+
+**Reading.** A measured value with a time: inlet temperature, power draw, a
+cooling unit's alarm state. Readings older than their staleness limit are
+treated as absent.
+
+## Estimating a replica's draw
+
+What placement reserves has to be known before the replica runs, so it is an
+estimate first and a measurement later:
+
+1. **Declared.** A profile may say `power_watts`. The operator's figure wins.
+2. **Derived.** Otherwise GPU profiles take each card's power limit from NVML
+   (the agent already reads the cards; the power limit is one more field) plus
+   a per-node share for CPU, memory and fans; CPU profiles take their reserved
+   cores times a per-core figure for the node's CPU model.
+3. **Measured.** Once replicas run, the agent reports their cards' actual draw
+   (NVML) and, where the BMC allows, the node's (Redfish or IPMI DCMI). A
+   domain's headroom is computed from the larger of what is reserved and what
+   is measured, so an estimate that was too low is corrected by the room it
+   stops offering, not by an outage.
+
+## Decisions
+
+### Power
+
+- **Hard limit.** A placement that would take any domain past `cap × (1 −
+  margin)` is rejected, with the domain and the figures: *rack r12-07: 14.2 of
+  16.0 kW reserved, replica needs ~2.8, margin 10%*.
+- **Feeds.** For a domain with two feeds the cap is the smaller feed's rating
+  times its derating (typically 80%), not the sum.
+- **Already over.** A domain found over its cap, because the estimate was low
+  or the cap was lowered, gets no new placements, and an alert. It is not
+  emptied automatically; shedding power is the facility's decision, and
+  stopping inference to do it is ours only if asked (see shedding below).
+
+### Temperature and cooling
+
+Thresholds are per domain with defaults, and each condition needs to hold for a
+while before it changes, in both directions, so a single reading neither
+starts nor ends anything:
+
+| Condition | Enters when | Placement |
+|---|---|---|
+| `ok` | below the warm threshold for 5 min | normal |
+| `warm` | inlet above the warm threshold for 2 min, GPU temperatures near throttling, or any GPU thermally throttled | allowed, scored down: other domains first |
+| `hot` | inlet above the hot threshold for 2 min, a cooling unit in alarm, or a sustained share of the domain's GPUs thermally throttled | no new placements, as a cordon would, but automatic and reversible |
+| `unknown` | no fresh reading | as `ok`, and reported as unknown |
+
+**Shedding** (optional, off by default): a domain `hot` for longer than its
+shed delay has load moved off it using step 2's machinery. The lowest class goes
+first, and replicas above their model's minimum go before any at it. A critical
+minimum is never touched, and moves are rate limited to one replica per domain
+per interval. The replica is placed elsewhere first where there is room, so
+shedding moves work rather than dropping it. GPU power capping (`nvidia-smi
+-pl`) is a gentler alternative worth measuring against shedding: less
+throughput, no restart.
+
+### Spreading
+
+Replicas of one model are spread across racks, and across PDUs, the way they
+are spread across nodes today. This is a preference, dropped when it is the only
+thing standing in the way, as node spreading already is.
+
+### Scoring
+
+Placement today filters, then takes the first candidate in a fixed order
+(fewest free GPUs for GPU profiles, fewest replicas for CPU ones, nodes holding
+the weights first). Facility signals are better as a score than as more
+filters. Candidates that pass the hard limits are ordered by:
+
+1. condition: `ok` before `warm`;
+2. power headroom left in the rack after this replica, more first, so load
+   levels across racks;
+3. the existing order.
+
+Once measurements include power, step 4's profile preference can use tokens per
+watt instead of tokens per GPU where both are known.
+
+## Where the data comes from
+
+A `FacilitySource` interface, implemented several ways, chosen in
+`server.yaml`:
+
+```go
+type FacilitySource interface {
+    // Domains is what the source knows: budgets, thresholds, members.
+    Domains(ctx) ([]Domain, error)
+    // Readings is the latest value of each signal, per domain.
+    Readings(ctx) ([]Reading, error)
+}
+```
+
+- **Static file.** Domains, caps and thresholds, written by hand or generated
+  from a DCIM export. Enough for power budgets with no live data at all.
+- **Prometheus.** DCIM, PDU (SNMP exporter) and BMC (Redfish exporter)
+  readings usually end up in Prometheus. The source runs configured instant
+  queries and maps their labels onto domains:
+
+  ```yaml
+  facility:
+    source: prometheus
+    prometheus: http://monitoring:9090
+    signals:
+      inlet_temp_c:
+        query: max by (rack) (dcim_rack_inlet_temperature_celsius)
+        domain_label: rack
+        stale_after: 2m
+      power_watts:
+        query: sum by (pdu) (pdu_outlet_power_watts)
+        domain_label: pdu
+      cooling_alarm:
+        query: max by (row) (crac_alarm)
+        domain_label: row
+  ```
+- **Agents (built, step 0).** What each node reads about itself, every ten
+  seconds at most, sent with its heartbeat:
+  - **GPUs, from nvidia-smi:** temperature, memory (HBM) temperature where the
+    driver reports it, power draw, the enforced power limit, and what is
+    holding the clocks down now — `sw_power_cap`, `hw_slowdown`, `sw_thermal`,
+    `hw_thermal`, `power_brake`.
+  - **CPUs, from hwmon** (what lm_sensors reads, without it installed): the
+    package temperature, or the hottest core; and the board's inlet or ambient
+    sensor where it has one (`SYSTIN`, `Inlet`, `Ambient`).
+  - **CPU power, from RAPL:** the packages' energy counters over time,
+    subdomains not counted twice, counter wrap handled.
+
+  A machine without a sensor reports nothing for it, never a zero; a VM has
+  none of them. Published as `llmgw_gpu_temperature_celsius`,
+  `llmgw_gpu_memory_temperature_celsius`, `llmgw_gpu_power_watts`,
+  `llmgw_gpu_power_limit_watts`, `llmgw_gpu_throttled{reason}`,
+  `llmgw_node_cpu_temperature_celsius`, `llmgw_node_cpu_power_watts` and
+  `llmgw_node_inlet_temperature_celsius`, and on each node's page.
+
+  What they can and cannot say:
+  - **Throttling is the strongest signal there is.** A temperature is a proxy;
+    a card holding its clocks down is the fact that invalidates its measured
+    capacity, and it needs no threshold to be chosen.
+  - **Measured power** is GPU draw plus CPU packages; memory, disks, fans and
+    power-supply losses are not in it, so a node's draw is that plus an
+    estimated remainder. A rack's is its llmgw nodes' plus a declared figure
+    for everything else in it: switches and storage report nothing here.
+  - **CPU and GPU temperatures follow the load, not the room.** A hot rack is
+    inferred from many nodes running hot at comparable load, or from
+    throttling, not from one node's CPU. Only a board inlet sensor reads the
+    air, and most boards have none.
+  - **No budget comes from a sensor.** A reading is a draw; the cap it is
+    measured against still comes from the facility.
+- **Simulated.** Readings from a script, for tests and for rehearsing a
+  thermal event on a cluster without facility data.
+
+## Safety
+
+- **Missing data never blocks.** A source that is down, a reading that is stale,
+  a node without location labels: placement behaves as it does today, and says
+  the data is missing (a metric, a line on the cluster page). Facility
+  awareness can make placement more careful, never impossible.
+- **Hysteresis and hold times** on every condition, so a noisy sensor does not
+  flap a rack between allowed and forbidden.
+- **Observe-only first.** A mode where every decision is computed, logged and
+  shown ("would have refused: rack r12-07 hot") but nothing is enforced, run
+  for long enough to trust the thresholds before turning enforcement on.
+- **Manual override.** `llmgw ctl facility hold RACK` and `release RACK`, for the
+  facility team to mark a domain unavailable, or to clear a condition they know
+  to be a sensor fault.
+- **Shedding is opt-in**, per domain or cluster-wide, and rate limited.
+
+## What people see
+
+- `explain-placement` and headroom name the domain: *rack r12-07 is hot (inlet
+  31°C for 6 min)*, *pdu r12-07-a: 7.1 of 7.4 kW*.
+- `llmgw ctl facility` lists domains with budget, reserved, measured and
+  condition. The cluster page gets a rack summary beside the nodes table.
+- Metrics: `llmgw_domain_power_budget_watts`, `_reserved_watts`,
+  `_measured_watts`, `llmgw_domain_condition{condition}`,
+  `llmgw_domain_reading_age_seconds`; alerts for a domain near its cap, a
+  domain hot, a domain over its cap, and facility data gone stale.
+
+## Rollout
+
+Each step ships on its own and is useful without the next:
+
+0. **Node sensors, observe-only (built).** GPU heat, power and throttling,
+   CPU heat and power, inlet air where the board has a sensor; published and
+   shown, no decision changed.
+1. **Location and spreading.** Well-known labels, domains from labels, replicas
+   spread across racks and PDUs, locations on the node views. No facility data
+   needed.
+2. **Power budgets from a static file.** Caps and margins, estimated draw per
+   replica (declared or derived), hard limit in placement and headroom.
+3. **Readings, observe-only.** The Prometheus source beside the agents',
+   conditions with hysteresis (throttling first among them), measured power
+   folded into headroom; decisions shown, not enforced.
+4. **Thermal enforcement.** `warm` scored down, `hot` refused, manual hold and
+   release.
+5. **Shedding and tokens per watt.** Opt-in shedding through step 2's
+   preemption; measured power in profile preference.
+
+## Testing
+
+- The simulated source drives unit tests of every condition transition, the
+  hysteresis, staleness, and the placement outcomes, with mutation checks as
+  for steps 1–4.
+- Property tests: whatever the readings, placement never exceeds a declared
+  cap with fresh data, and never refuses everything because data is missing.
+- On the PoC cluster, with no facility data: step 1 with labelled VMs, step 2
+  with a static file of made-up caps, steps 3–4 with the simulated source
+  replaying a thermal event.
+- Before enforcement anywhere real: a period in observe-only mode against the
+  site's actual data.
+
+## Open questions
+
+- **What the sites have.** Which DCIM, which PDUs, which BMCs, and whether any
+  of it already reaches a Prometheus. The source design assumes Prometheus is
+  the common ground; a site without it needs another source.
+- **Who owns the numbers.** Caps, deratings and thresholds belong to the
+  facility team. Whether they edit llmgw's file, or llmgw reads theirs.
+- **Phase balance.** Three-phase PDUs have per-phase limits; whether placement
+  should see phases or only the PDU total.
+- **Cooling granularity.** Cooling is often per row or per aisle, not per rack;
+  the domain tree allows either, but the defaults should match what sites
+  actually measure.
+- **Power capping instead of shedding.** Whether to lower GPU power limits in a
+  warm rack rather than move replicas, and how that interacts with measured
+  capacity from step 4, which would no longer hold at the lower limit.

docs/install.md

@@ -1,7 +1,8 @@
 # Installing llmgw from nothing
 
 Written for whoever is standing up the cluster: you have servers, you have root
-on them, and nothing is installed yet. By the end there is a control plane, a
+on them, and nothing is installed yet. For only the commands, read
+[quick-install.md](quick-install.md) first; this page explains them. By the end there is a control plane, a
 node in the fleet, and a model answering requests.
 
 It assumes Debian or Ubuntu and adjusts nothing for other distributions beyond
@@ -86,7 +87,24 @@ CGO_ENABLED=0 go build -o llmgw ./cmd/llmgw
 
 ## 2. The control plane
 
-On the machine that will be the control plane, as root:
+**The short way is the screens.** On the machine that will be the control
+plane, run `sudo llmgw` with nothing after it. With nothing installed it asks
+whether this is one machine or three for high availability; for three, it asks
+for the other two — address, user, and an ssh key, the ssh agent, or a
+password, and the sudo password if sudo asks for one; or, for a machine
+nothing may log into, a join code to run there with `sudo llmgw join <code>`,
+which dials out to this one on port 7444 — checks all three before
+installing anything (system, room, clock, ports, internet, whether they reach
+one another), shows the plan, and installs. Passwords are held in memory for
+the run only; a sudo password lets the user sudo without one until the install
+finishes, and is then taken back. At the end it shows the administrator's
+password and the command that does the same from a script.
+
+Run again later, `sudo llmgw` is the console: the cluster's state, refreshed,
+and what to do next — add a node, replicate the store. A setup that stopped
+halfway opens the setup screens again, which finish it.
+
+The rest of this section is the same by command, as root:
 
 ```
 sudo llmgw server init --admin you@example.com
@@ -104,11 +122,36 @@ That is the whole step. In order, it:
    and promoted to administrator — the password is generated and **printed
    once**
 6. installs the systemd unit, starts it, and waits until `/health` answers
+7. runs a model mirror beside it (`/srv/models`, bytes on `:8081`, its
+   management API on loopback) and points the control plane at it
+8. makes this machine a node of its own control plane — on its GPUs when
+   `nvidia-smi` finds any, on CPU otherwise — and waits for it to install its
+   engine
+9. fetches Qwen2.5 0.5B into the mirror and serves it on this node, so the
+   first thing anyone opens `/chat` to find is a model that answers
+
+`--no-mirror`, `--no-node` and `--no-model` leave the last three out. An NVIDIA
+card with no working driver is reported rather than silently ignored; install
+the driver and run init again to serve on it.
 
 Add `--postgres` to keep accounts and keys in PostgreSQL rather than SQLite:
 init installs it and sets it up, and on a machine already running on SQLite it
 moves everything across. See [PostgreSQL](#postgresql-when-sqlite-is-no-longer-the-right-shape).
 
+Or `--patroni`, given two machines (or more), to keep it in PostgreSQL
+replicated across them, so losing one loses nothing:
+
+```
+sudo --preserve-env=SSH_AUTH_SOCK llmgw server init --patroni ubuntu@10.0.0.149 --patroni ubuntu@10.0.0.150
+```
+
+Over ssh with passwordless sudo, as `node add` works, it installs PostgreSQL,
+Patroni and etcd there — this machine is etcd's third vote — and moves the
+store already in use, SQLite or PostgreSQL, into the cluster. A standby is
+promoted within about thirty seconds of the primary being lost, and the
+control plane follows it by itself. `llmgw ctl status` says who leads; see
+[deploy/ha](../deploy/ha/README.md) for what it sets up.
+
 Every step looks before it acts. Running it again finishes an install that
 stopped halfway and changes nothing on one that did not; it never overwrites
 `server.yaml`, the authority, or an existing account's password.
@@ -307,32 +350,26 @@ hardware to it.
 
 ### NVLink, if your GPUs have it
 
-The published binary cannot see NVLink topology. NVML is the only source of it,
-its bindings need cgo, and the portable binary is built without cgo on purpose
-so it runs on a node whose libc is older than the builder's.
+The agent reads the topology from `nvidia-smi topo -m`: which cards NVLink
+joins, each card's NUMA node, and which network adapter is nearest each. A
+tensor-parallel group is then the set of cards with the most NVLink between
+them — on a PCIe box where bridges pair specific cards, a group straddling two
+bridges falls back to PCIe and runs several times slower, and nothing else
+would report it.
 
-Without that topology, a tensor-parallel group takes the lowest free GPU
-indices. On an SXM board with NVSwitch that is correct, because every card is
-equidistant. On a PCIe box where NVLink bridges pair specific cards, a group
-straddling two bridges falls back to PCIe and runs several times slower — and
-nothing reports it, because from the outside it is simply a replica that is
-slow.
-
-For those nodes, install the `-nvml` build published beside the portable one:
+On an NVSwitch machine, `nvidia-fabricmanager` has to be running, at the
+driver's version, or CUDA will not start on its GPUs. On an NVL72 rack or an
+NVLink Switch system, the GPUs of several nodes form one domain, and a replica
+can run tensor parallelism across its nodes — which needs `nvidia-imex` and an
+IMEX channel as well. [fabrics.md](fabrics.md) has the list, and `llmgw doctor`
+checks it:
 
 ```
-sudo install -m0755 llmgw-$VERSION-linux-amd64-nvml /usr/local/bin/llmgw
+sudo llmgw doctor
 ```
 
-It is dynamically linked, so it needs a libc no older than the builder's and the
-NVIDIA driver present. Check it took effect with:
-
-```
-llmgw ctl nodes --json | grep -c nvlink_peers    # 0 means the topology is unknown
-```
-
-`llmgw_gpu_nvlink_peers` in the metrics says the same thing across a fleet: a
-flat zero everywhere means the topology is unknown rather than absent.
+A `-nvml` build is still published beside the portable one; it reads NVLink
+from NVML directly, and needs a libc no older than the builder's.
 
 ### InfiniBand, if you have it
 
@@ -342,48 +379,112 @@ ibstat            # must list your HCAs with State: Active
 ls /sys/class/infiniband
 ```
 
-The agent checks what you declare against what the node has. An adapter that is
-not there, or that has no ACTIVE port — physically linked but with no subnet
-manager, which survives a reboot and looks like a software problem — takes that
-fabric out of service on this node, loudly, in the log. Without that check the
-replica starts, passes its health checks, and hangs on its first all-reduce.
+Each fabric needs a subnet manager, and should have a standby; `nvidia_peermem`
+should be loaded for GPUDirect RDMA. You name each fabric when you onboard the
+node, and llmgw checks the name against what the adapters report: every port on
+one fabric sees the same subnet managers, so two independent fabrics given one
+name are placed as two, and the cluster page says so. An adapter that is not
+there, or whose port goes down, takes that fabric off the node within seconds,
+and gives it back when the port returns.
 
-Fabric membership is configuration, not discovery, and there is no way around
-that: OpenSM's default subnet prefix is the same on every fabric, so a matching
-prefix proves nothing and a wrong guess produces a replica that starts, passes
-health checks, and then hangs on its first all-reduce. You name the fabric when
-you onboard the node, and two nodes are only in the same fabric if you said so.
+[fabrics.md](fabrics.md) covers declaring several fabrics, what the cluster and
+fabric pages show, and what to look at when a replica spanning nodes will not
+run. Run `llmgw doctor` on the node before it joins.
 
-### An engine
+### Engines
 
-Pick what you will actually serve. For a first cluster, llama.cpp is the least
-trouble and runs without a GPU.
+Nothing to do: the agent installs them once the node joins, under
+`/opt/llmgw/engines`, and checks each one runs before the node offers it.
 
-```
-# llama.cpp, from a release build
-sudo apt-get install -y libgomp1          # it will not start without this
-sudo mkdir -p /opt/llama.cpp
-# unpack a llama.cpp release into /opt/llama.cpp
-/opt/llama.cpp/llama-server --version
+| Engine | On | How |
+|---|---|---|
+| llama.cpp | every node | the release build for its processor and GPU — CUDA 13.3, CUDA 12.8 (x64), or Vulkan for GPUs and drivers the CUDA builds do not cover; the CPU build only where there is no NVIDIA GPU — checked against GitHub's checksum and seen to list the GPU |
+| vLLM | nodes with an NVIDIA GPU it can run on | a Python environment built by uv from PyPI, about 8 GB |
+| SGLang | only when you ask for it | the same way as vLLM |
+
+vLLM is resolved as PyPI stood a day after that vLLM was published, so every
+node gets the same dependencies whenever it is built, and installed with
+`--require-hashes` against the sha256 PyPI publishes. The resolved list stays in
+the environment as `llmgw-requirements.txt`. It needs:
+
+- **the build its hardware runs**, chosen by the node. Driver 580 and newer:
+  the CUDA 13 build from PyPI. Driver 525 to 579: the same version built for
+  CUDA 12.9, from vLLM's GitHub release (checked against the digest GitHub
+  publishes) with PyTorch's CUDA 12.9 torch. Both have kernels for Turing
+  (7.5) and newer, 8.0 and newer on aarch64. **A V100** (Volta, 7.0) gets
+  **vLLM 0.19.1**, the newest built for it — no build from 0.20 on has its
+  kernels — and the cluster page shows it running that version; run `llmgw
+  ctl engine-check vllm` there once, since llmgw's flags are checked against
+  the current one. Pascal and older run no vLLM. Whatever is chosen, the
+  install ends by computing something on the GPU; a build that cannot is
+  passed over for the next;
+- **about 25 GB free** under the engines directory while it installs;
+- **gcc**, which Triton builds its kernel launcher with. The agent installs it.
+
+A node with no GPU skips vLLM and says nothing about it. A node that has a GPU
+but cannot run it — an old driver, a V100, a small disk — says why on the
+cluster page. `llmgw doctor` checks the same before the node joins.
+
+The versions are the control plane's, in `server.yaml`:
+
+```yaml
+engines:
+  vllm: "0.30.0"      # the default; "" leaves vLLM to you
+  sglang: "0.5.20"    # SGLang only when named
 ```
 
-That `libgomp1` line is not padding. A `llama-server` binary missing it exists,
-is executable, and cannot start — and the node reports `ready` throughout.
+A changed version reaches every node when it next connects. To use an engine
+you installed yourself on a node, name its path in that node's `agent.yaml`
+(`engines.llamacpp.bin`, `engines.vllm.bin`, `engines.sglang.python`): the agent
+then installs none of its own beside it.
 
-For vLLM, a virtualenv on the node:
+#### Choosing a node's vLLM build
+
+Which build a GPU node installs is chosen on the cluster page, in the **vLLM
+build** column — *auto* (what its hardware runs, the default), *CUDA 13*,
+*CUDA 12.9*, or *none* — or from the command line:
 
 ```
-sudo apt-get install -y python3-venv
-sudo python3 -m venv /opt/venvs/vllm
-sudo /opt/venvs/vllm/bin/pip install vllm
-/opt/venvs/vllm/bin/vllm --version
+llmgw ctl engine-build gpu07 vllm cu129
 ```
 
-Check the engine still takes the flags llmgw builds for it, which drift between
-releases:
+The choice is kept by the control plane and the node is told at once; it
+installs the build in the background, and the column shows how that stands.
+
+#### Different GPUs, different engines
+
+A cluster of two GPU generations may need two builds of an engine, or one on
+only some nodes. `engine_overrides` picks nodes — by a GPU class they have, by
+id, or by labels, every one given having to match — and lays engines over the
+cluster's for them, in order:
+
+```yaml
+engines:
+  vllm: "0.30.0"              # CUDA 13: Turing and newer
+engine_overrides:
+  - gpu_class: a10g           # pin a class to a version you have tested
+    engines: { vllm: "0.29.0" }
+  - labels: { role: gguf }    # nodes meant for llama.cpp alone
+    engines: { vllm: "" }     # none
+```
+
+A node told nothing it can run of an engine joins without it: `llmgw doctor`
+and `node add` warn, and the node serves what it does run. To keep an engine
+off a node for good, whatever the control plane says, add the node with
+`llmgw node add --disable-engine vllm`, which writes `engines.disabled: [vllm]`
+into its `agent.yaml` and leaves vLLM out of doctor's checks. That is narrower
+than `--ignore-checks`, which passes over every other problem as well.
+
+A llama.cpp CUDA build is checked for the GPU it is meant for: installing it
+asks it which devices it sees, and a replica placed on GPUs is not ready until
+its process holds memory on each of them. A build missing a CUDA library fails
+there, naming the library, rather than serving on the CPU.
+
+After changing a version, check it still takes the flags llmgw builds for it,
+which drift between releases:
 
 ```
-llmgw ctl engine-check vllm --bin /opt/venvs/vllm/bin/vllm
+llmgw ctl engine-check vllm --bin /opt/llmgw/engines/vllm/0.30.0-cu13/bin/vllm
 ```
 
 ### Weights
@@ -470,40 +571,44 @@ bf16 is 141 GB and the same model at q4 is 43 GB.
 
 ## 5. Adding the node
 
-Two halves, because they need different access: building the machine needs the
-hypervisor, and issuing its identity needs the certificate authority. Where one
-person holds both — the common case — one command does it:
+On the control plane, which holds the certificate authority:
 
 ```
-llmgw-add-node \
-  --name llmgw-node01 --ip 10.0.0.141/24 \
-  --server https://10.0.0.140:7443 \
-  --cores 16 --memory 131072 --disk 200 \
-  --fabric ib-a:mlx5_0,mlx5_1 \
-  --llamacpp-bin /opt/llama.cpp/llama-server
+sudo llmgw node add ubuntu@10.0.0.141 --fabric ib-a:mlx5_0,mlx5_1
 ```
 
-**The VM name, the hostname, the node id and the certificate's common name are
-all the same string.** They do not have to be, and every time they are not,
-somebody spends an afternoon working out which of the four a message refers to.
+It logs in to the machine over SSH as that user, who needs passwordless sudo
+there, and:
 
-For a machine that already exists — bare metal, or a VM you built yourself — skip
-provisioning and issue the identity only:
+1. installs llmgw: the release for the machine's architecture, downloaded by
+   the machine and checked against the release's checksums, or this machine's
+   copy when it cannot download;
+2. runs `llmgw doctor` there and stops on any problem, printing the fix. Run it
+   again once fixed, or add `--ignore-checks`;
+3. issues the node's certificate and installs it, with the cluster's root,
+   trusted system-wide;
+4. writes `agent.yaml` — only what the node cannot find for itself — and the
+   systemd unit, and starts the agent;
+5. waits for the node to join and to install llama.cpp (`--wait` also waits
+   for vLLM, `--no-wait` for neither).
 
-```
-llmgw-onboard-node \
-  --node llmgw-node01 --addr 10.0.0.141 \
-  --server https://10.0.0.140:7443 \
-  --fabric ib-a:mlx5_0,mlx5_1 \
-  --llamacpp-bin /opt/llama.cpp/llama-server
-```
+In a terminal it draws this as a checklist and asks what a script cannot: which
+fabric each group of the node's active adapters is on, when `--fabric` was not
+given, and whether to go on past a problem doctor found.
 
-Onboarding needs SSH from the CA host to the node. It waits for SSH rather than
-failing on it, and learns the host key of a machine it has not seen — a new
-machine spends its first minute regenerating host keys and restarting sshd.
+**The node id is the machine's hostname**, and the address its agent dials is
+this machine's on the route to it; `--node` and `--server` override them. The
+VM name, the hostname, the node id and the certificate's common name being the
+same string saves an afternoon each time they would differ.
 
-It also checks that a configured engine actually runs, and says so loudly when
-it does not. That check exists because the failure is otherwise invisible.
+**SSH runs as root under sudo**, with root's keys. To use yours, log in to the
+control plane with `ssh -A` and run `sudo --preserve-env=SSH_AUTH_SOCK llmgw node
+add …`. It waits for SSH rather than failing on it, and learns the host key of
+a machine it has not seen — a new machine spends its first minute regenerating
+host keys and restarting sshd.
+
+Running it again on a node is safe: a fresh certificate and configuration, and
+the engines it runs are adopted by the restarted agent.
 
 Within a few seconds:
 
@@ -649,6 +754,74 @@ llmgw ctl explain-placement qwen2.5-coder-3b-instruct
 which names every candidate and why each was rejected, rather than leaving you
 to infer it.
 
+### LoRA adapters
+
+A llama.cpp model can carry LoRA adapters, each asked for by its own name as
+though it were a model:
+
+```yaml
+- name: qwen2.5-coder-7b-instruct
+  source: http://mirror:8081/qwen2.5-coder-7b-instruct/5d567d26/model.gguf
+  adapters:
+    - name: coder-sql
+      source: http://mirror:8081/coder-sql/v1/adapter.gguf
+      scale: 1.0          # optional; 1 when left out
+  profiles: [...]
+```
+
+Every replica of the model fetches every adapter and loads it at start, applied
+to nothing. A request for `coder-sql` goes to the model's replicas with the
+adapter applied to it alone; a request for the model gets the model. Adapter
+names share the namespace of models and aliases, appear in `/v1/models`, and
+follow key allowlists on their own: a key allowed `coder-sql` reaches that
+adapter and not the model under it, while a key allowed the model reaches its
+adapters too. Adding or removing an adapter is a change to the model; running
+replicas load the new set when restarted (`llmgw ctl restart --model M`).
+Besides the mirror page's model card, adapters are attached from the command
+line, which reads the file first and refuses one that is not an adapter for
+this model's architecture:
+
+    llmgw ctl model adapters coder --add coder-sql=http://mirror:8081/sql-lora/v1/sql.gguf@0.8
+    llmgw ctl model adapters coder --remove coder-sql
+
+The adapter file must be a GGUF LoRA made for this model — llama.cpp's
+`convert_lora_to_gguf.py` makes one from a PEFT adapter — and not a model with
+the adapter already merged in, which is what most "LoRA GGUF" repositories hold.
+
+### Models served outside the cluster
+
+A model can be an OpenAI-compatible API elsewhere — a hosted provider, or
+another gateway — rather than replicas here. Nothing is placed for it; the
+router sends its requests on, with the provider's key and the provider's name
+for the model:
+
+```yaml
+- name: cloud-mini
+  source: external
+  external:
+    url: https://api.openai.com          # without /v1
+    model: gpt-4o-mini                   # what the provider calls it
+    api_key_file: /etc/llmgw/secrets/openai.key
+```
+
+Named in another model's `fallbacks`, it is used only when the cluster has no
+capacity left for that model:
+
+```yaml
+- name: qwen2.5-coder-7b-instruct
+  fallbacks: [cloud-mini]
+  ...
+```
+
+The key is read from its file on the control plane, never stored in the
+catalog, and a key rotated in the file is used within seconds. A model whose key
+file cannot be read is not offered. Everything else applies as to any model:
+prices and budgets, rate limits, usage records, and key allowlists.
+
+**Its prompts leave the cluster.** Decide who may send them: a key or project
+whose allowlist leaves the external model out cannot reach it, as a fallback or
+by name. The cluster page and `ctl models` mark it `external`.
+
 ### How many replicas, and how much each one carries
 
 Replica count is changed while the cluster runs, without editing the
@@ -887,88 +1060,165 @@ refuses every request.
 
 ## Upgrading
 
-One binary on every machine, so an upgrade is a copy and a restart. The part
-worth care is the copy, not the order.
+On the control plane:
+
+```
+sudo llmgw ctl upgrade v0.10.4
+```
+
+It upgrades this machine first — fetching the release for its platform,
+checking it against the release's checksums, keeping the old binary as
+`/usr/local/bin/llmgw.bak-pre-<version>`, restarting `llmgw-server` and waiting
+for every node to reconnect — and then each node in turn. A node's agent is told
+the version over the stream it already holds, fetches and checks the release
+itself, swaps it in and restarts; no ssh into the nodes is needed. Release
+candidates (`v0.10.4-rc.2`) work the same way.
+
+A node counts as done when it is back on the new version, ready, with as many
+ready replicas as before. One that is not within `--timeout` (three minutes)
+stops the upgrade there, so a bad release reaches one node rather than all of
+them; `llmgw ctl logs <node>` says why, and the node keeps its previous binary
+beside the new one. Nodes already on the version are skipped, so running the
+command again resumes where it stopped. `--nodes a,b` limits it to some nodes,
+and `--skip-control-plane` leaves this machine alone.
 
 **Nothing stops serving.** Engines are started with `KillMode=process`, so they
 keep running when the agent restarts, and the agent reports them back on its
 next connection. The control plane holds no replica state across its own
-restart either — it rebuilds the fleet from what the agents report. A rolling
-upgrade of the whole cluster interrupts no request that is not in flight at the
-moment of the restart.
+restart either — it rebuilds the fleet from what the agents report.
 
 **A mixed fleet is fine while you roll.** The agent protocol only ever gains
 fields, and both sides treat an absent one as unknown rather than as a default.
-Order does not matter; finish the roll the same day anyway, because
-`ctl nodes` reports each agent's version and a fleet that disagrees with itself
-is one more thing to hold in your head.
+Agents older than v0.10.4 cannot upgrade themselves: move those by hand once,
+installing the release binary and restarting `llmgw-agent`, and every upgrade
+after that is the command above.
+
+**The mirror is upgraded on its own host**, since it is not an agent and its
+service cannot replace its own binary:
 
 ```
-VERSION=v0.9.0
-# copy to every machine first, install nowhere yet
-for h in 10.0.0.140 10.0.0.141 10.0.0.142 10.0.0.146; do
-  ssh $h 'cat > /tmp/llmgw-new; stat -c %s /tmp/llmgw-new' < llmgw-$VERSION-linux-amd64
-done
+sudo llmgw upgrade v0.10.5 --service llmgw-mirror
 ```
 
-**Check every size before installing anything.** This is not defensive writing:
-on a host whose login shell prints anything, `scp` fails outright, and the
-obvious workaround — `ssh -n host 'cat > file'` — points stdin at `/dev/null`
-and writes a *zero-byte file* while reporting success. Installing that on a
-fleet takes the fleet down. Compare the number on each machine with the local
-one, then:
-
-```
-sudo install -m0755 /tmp/llmgw-new /usr/local/bin/llmgw
-sudo systemctl restart llmgw-agent     # or llmgw-server, or llmgw-mirror
-llmgw --version
-```
+The same checks — the release's checksums, the old binary kept beside the new —
+then a restart of the unit, waiting for it to stay up. A mirror older than
+v0.10.5 does not have the command: install the release binary by hand once.
 
 Then confirm what you have, rather than assuming:
 
 ```
-llmgw ctl nodes       # every agent back, LAST SEEN in seconds
+llmgw ctl nodes       # every agent on the new version, LAST SEEN in seconds
 llmgw ctl replicas    # same replicas, adopted rather than replaced
 ```
 
 A replica may come back marked **`ready (stale profile)`**. That means it is
 running a profile that has since been edited — the reconciler leaves a serving
 replica alone on purpose, because churning one costs more than the difference.
-Restart it when you want the new profile to take effect; the flag is there so
-the choice is yours rather than silent.
+`llmgw ctl restart <replica>` replaces it with one on the new profile, starting
+the new one before stopping the old. Where the old one is all that stands in
+the new one's way — a single GPU — it is stopped first, at once, and the model
+is unserved while the new one loads; where the new profile fits nowhere even
+then, the restart is refused and the old one keeps serving, which `ctl
+replicas` shows as `restart refused:` with the reason. `--in-place` asks for
+stop-then-start whatever room there is.
+
+### Signed releases
+
+Every install above checks a release against its published checksums, which
+proves the download is what the server holds — not that the server holds what
+was built. Signing closes that: CI signs the checksums, and a binary that
+trusts a key installs only releases signed with it. Off until a key exists;
+turning it on, once:
+
+```
+llmgw release keygen
+```
+
+1. Put the **private** key in the repository's Actions secret
+   `RELEASE_SIGNING_KEY`. From the next tag, releases carry
+   `checksums.txt.sig`.
+2. Put the **public** key in `internal/release/trusted.pub` and release. Every
+   binary built from then on carries it and refuses releases that are not
+   signed with it — `ctl upgrade`, agents upgrading themselves, and
+   `llmgw upgrade` alike.
+
+In that order: a build that trusts a key refuses unsigned releases, so the
+secret has to be signing before the key is trusted. The first trusting release
+is itself installed by a binary that trusts nothing; every one after is checked
+by the one before. `LLMGW_RELEASE_KEY` overrides the built-in key, for rotating
+it. The onboarding script fetches with `curl` and checks checksums only; a node
+is checked from its first upgrade on.
 
 ## Backing up
 
-Three things, in descending order of how badly you want them.
+`llmgw server backup` writes one archive holding everything a control plane
+cannot be rebuilt without:
 
-**The certificate authority.** `ca-key.pem`, `/etc/llmgw/pki` unless you kept it elsewhere (step 2).
-Losing it does not stop the cluster, and it means every node identity has to be
-reissued the next time you touch one. Nothing in llmgw can reconstruct it.
+- **the store** — users, API keys, the tenancy tree, limits and budgets, usage
+  history, models added at runtime, measured capacity, and the secret every
+  engine key is derived from (`pg_dump` in its custom format for PostgreSQL, a
+  consistent `VACUUM INTO` snapshot for SQLite; both are safe against a running
+  gateway);
+- **`/etc/llmgw`** — the configuration, the store's address, and the
+  certificate authority's key (`pki/`). Nothing in llmgw can reconstruct that
+  key; losing it means reissuing every node's identity.
 
-**The store.** Users, API keys, organisations, teams, projects, every limit and
-budget, all usage history, models added at runtime, and measured capacity.
+Archives go to `/var/backups/llmgw-server`, readable by root only because of the CA
+key, and the newest 14 are kept. Run it nightly with the timer:
 
 ```
-# SQLite
-sqlite3 /var/lib/llmgw/llmgw.db ".backup '/backup/llmgw-$(date +%F).db'"
-
-# PostgreSQL
-pg_dump --format=custom llmgw > /backup/llmgw-$(date +%F).dump
+sudo install -m 0644 deploy/systemd/llmgw-backup.service deploy/systemd/llmgw-backup.timer /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl enable --now llmgw-backup.timer
+sudo systemctl start llmgw-backup.service     # one now, to see it work
+ls -l /var/backups/llmgw-server
 ```
 
-`.backup` and `pg_dump` are both safe against a running gateway. Copying the
-SQLite file with `cp` while it is being written is not.
+The control plane publishes when the newest archive was written as
+`llmgw_backup_last_success_timestamp_seconds`, and **BackupTooOld** fires when
+it is more than 26 hours old — including when there has never been one.
+
+**Copy the archives off the machine.** They sit on the disk they back up and do
+not survive losing it. Anything that copies files will do — rsync from the
+mirror, a NAS mount, an offsite bucket — as long as it keeps the archives
+readable by root only.
 
 **Keys are stored hashed and cannot be recovered from a backup or from
 anywhere else.** Restoring an old store restores which keys exist and who owns
 them, not their secrets. A key created after the backup is gone and has to be
-reissued — which is why the store is worth backing up on a schedule rather than
-before upgrades.
+reissued.
+
+### Restoring
+
+On a machine with llmgw installed (the same version or newer) and, for
+PostgreSQL, an empty database and role as `server init --postgres` would create
+them:
+
+```
+sudo systemctl stop llmgw-server
+mkdir /tmp/restore && sudo tar -xzf /var/backups/llmgw-server/llmgw-backup-<host>-<time>.tar.gz -C /tmp/restore
+cat /tmp/restore/MANIFEST                     # version and time it was taken
+
+# /etc/llmgw: configuration, secrets, certificates and the CA
+sudo cp -a /tmp/restore/etc/llmgw/. /etc/llmgw/
+
+# the store — PostgreSQL
+sudo -u postgres pg_restore --clean --if-exists --no-owner --role=llmgw -d llmgw /tmp/restore/store.pgdump
+# the store — SQLite
+sudo install -o llmgw -g llmgw -m 0600 /tmp/restore/store.sqlite /var/lib/llmgw/llmgw.db
+
+sudo systemctl start llmgw-server
+sudo rm -rf /tmp/restore
+```
+
+Running engines keep serving throughout: agents reconnect to the restored
+control plane and their replicas are adopted, and because the engine-key secret
+came back with the store, the router can still reach them.
 
 **The mirror's directory** is the one thing you can rebuild, by downloading it
-all again. Worth remembering that a pinned version may no longer be upstream to
-download: that is exactly what pinning it protected you from, and a backup is
-what makes the protection outlive the disk.
+all again. A pinned version may no longer be upstream to download: that is
+exactly what pinning it protected you from, and a copy of the directory is what
+makes the protection outlive the disk.
 
 ## Reclaiming disk
 
@@ -1037,6 +1287,8 @@ requests. On a slow model, set `timeouts.generation` higher.
   and what belongs on which port
 - `deploy/monitoring/README.md` — Prometheus, Grafana and the alerts, with what
   each one assumes about this system
+- `docs/fabrics.md` — InfiniBand fabrics and NVLink: what llmgw needs of them,
+  checking a node, and what to look at when a replica spanning nodes will not run
 - `docs/node-onboarding.md` — adding nodes in more detail, and what onboarding
   deliberately does not do
 - `docs/onboarding.md` — giving people accounts and keys
@@ -1050,16 +1302,27 @@ requests. On a slow model, set `timeouts.generation` higher.
   can see each other, seeding between them is the obvious next thing to build.
 - **A budget without a price never binds.** Set `default_price` or `pricing`
   (step 2) or every request costs zero, and a budget is something the dashboard
-  displays rather than something that stops anything. The quotas page says so
+  displays rather than something that stops anything. The Limits tab says so
   where it can, which is not the same as being told before you rely on it.
-- **Engines are yours to install.** Onboarding verifies one runs; it installs
-  nothing.
+- **A V100 runs vLLM 0.19.1**, not the version the rest of the cluster runs:
+  no build from 0.20 on has Volta kernels, and the node picks the newest that
+  has. llmgw's flags are checked against the current version, so run `llmgw
+  ctl engine-check vllm` on a V100 node once.
 - **Two control planes have not been run for long.** The leader election and
   agent failover work and there is a pattern in `deploy/ha/`, but every cluster
   this has been run on in anger has had one. A single control plane is the
   well-trodden path.
-- **Multi-node replicas have not been run on real InfiniBand.** The scheduling,
-  the fabric invariant and the rendezvous are exercised against simulated
-  hardware only. Single-node placement on real GPUs is the well-trodden path;
-  if you are the first to run a TP group across real cards, expect to find
-  something.
+- **Multi-node replicas have not been run on real InfiniBand or NVLink.** The
+  scheduling, the fabric identity, the maps and the launch are written against
+  the real tools' formats and tested against them, not yet run on hardware that
+  has them. Single-node placement on real GPUs is the well-trodden path; if you
+  are the first to run a TP group across real cards, run `llmgw doctor --json`
+  on each node first and keep the reports.
+- **Two engine setups are weaker than the rest.** An engine's key is passed
+  in its environment, where only root and its own user can read it, except
+  SGLang's, which takes it only as a flag: other users on the node can see it
+  in the process list. Mount `/proc` with `hidepid=2` on nodes where that
+  matters. vLLM's `launch_mode: ray` starts a Ray head without
+  authentication, so anyone who can reach its port can run code on the
+  node. The dashboard is off, but firewall the port to the other nodes in the
+  replica, or use the default `mp` launch mode, which needs no Ray at all.

docs/node-onboarding.md

@@ -4,24 +4,16 @@ Written for whoever runs the cluster.
 
 ## The short version
 
-Where one person holds both the hypervisor and the certificate authority —
-which is the common case — one command does the whole thing in about
-45 seconds:
+On the control plane:
 
 ```
-llmgw-add-node \
-  --name llmgw-node05 --ip 10.0.0.145/24 \
-  --server https://10.0.0.140:7443 \
-  --cores 8 --memory 32768 --disk 60 \
-  --llamacpp-bin /opt/llama.cpp/llama-server
+sudo llmgw node add ubuntu@10.0.0.145
 ```
 
-The VMID defaults to the last octet of the address, so `qm list` and
-`llmgw ctl nodes` stay in the same order. `--provision-only` and
-`--onboard-only` split it back into the two halves below when the two kinds
-of access belong to different people.
-
-The rest of this page is what that command does, and what it still does not.
+That is all a machine that already exists needs — bare metal, or a VM you
+built. [install.md](install.md#5-adding-the-node) says what it does. On
+Proxmox, the machine itself can be built first by the script below, on the
+hypervisor.
 
 ## The two halves
 
@@ -54,41 +46,31 @@ their working tree.
 
 ## 2. The identity
 
-On the host that holds the certificate authority — this is the part that cannot
-be delegated, because it issues the credential by which the control plane
-believes the node is who it says it is:
+On the control plane, which holds the certificate authority — this is the
+part that cannot be delegated, because it issues the credential by which the
+control plane believes the node is who it says it is:
 
 ```
-llmgw-onboard-node \
-  --node llmgw-node04 \
-  --addr 10.0.0.144 \
-  --server https://10.0.0.140:7443 \
-  --fabric ib-b
+sudo llmgw node add ilker@10.0.0.144 --fabric ib-b:mlx5_0,mlx5_1
 ```
 
-It issues a client certificate whose common name is the node id, installs it
-along with the CA root, trusts that root system-wide so other tooling on the
-machine stops treating llmgw's certificates as strangers, installs the binary
-and the unit, and starts the agent. The node appears in `llmgw ctl nodes` within
-a few seconds.
+It checks the machine with `llmgw doctor`, issues a client certificate whose
+common name is the node id (the hostname), installs it along with the CA root,
+trusts that root system-wide, installs the binary and the unit, starts the
+agent, and waits for the node to join.
 
-If it does not, the control plane's log says why. A certificate whose common
-name does not match the claimed node id is refused by design — that check is
-what makes a node's identity something it proves rather than something it picks.
+If it does not join, the agent's log on the node says why. A certificate whose
+common name does not match the claimed node id is refused by design — that
+check is what makes a node's identity something it proves rather than
+something it picks.
 
-**It needs ssh from the CA host to the node.** Provisioning copies that host's
-`authorized_keys` into the new machine, so a node built by the script above
-already satisfies it; an older machine may need the key added.
+**It needs SSH from the control plane to the node.** Provisioning copies the
+hypervisor's `authorized_keys` into the new machine; with `ssh -A` to the
+control plane and `sudo --preserve-env=SSH_AUTH_SOCK`, the same key that
+reaches the hypervisor reaches the node.
 
-It waits for ssh rather than failing on it, and learns the host key of a machine
-it has not seen (`accept-new`, which still refuses a *changed* key — the risk
-being guarded against is a substituted machine, not a new one). Both are needed
-to run the halves back to back: for the first minute of its life a new machine
-is still regenerating host keys and restarting sshd, so it refuses connections
-it accepted a moment earlier.
-
-It also checks that a configured engine actually runs, which is cheap here and
-expensive later — see below.
+`deploy/bootstrap/onboard-node.sh` is the script this replaced, for a machine
+holding an authority but not a current llmgw.
 
 ## The CPU model matters, and its default is wrong
 
@@ -116,32 +98,23 @@ The trade is live migration: a VM pinned to the host CPU cannot move to a
 machine with a different one. For a node whose job is arithmetic that is the
 right way round, and `--cpu` overrides it where it is not.
 
-## What onboarding does not do
+## Engines
 
-It gives a machine an identity and an agent. It does **not** install inference
-engines or model weights, and a node missing either looks healthy in
-`llmgw ctl nodes` right up until a replica is placed on it and the shard dies
-with "executable file not found".
+The script installs none, and does not need to: once the agent is connected,
+the control plane tells it which engines to have and it installs them itself
+— llama.cpp on every node, vLLM on every node with an NVIDIA GPU it can run
+on, SGLang where the control plane names a version. Each is checked to run
+before the node offers it, so a node never advertises an engine that cannot
+start. [install.md](install.md#engines) says what vLLM needs of the node.
 
-Onboarding does check that an engine you point it at can be executed, and says
-so loudly when it cannot. That check exists because the failure is invisible
-otherwise: `llama-server` on node05 was present, executable, and could not start
-for want of `libgomp1`, and the node reported `ready` in `llmgw ctl nodes` the
-whole time. Being told at onboarding beats finding out when a replica is placed
-and the reason is three logs away.
+`--llamacpp-bin`, `--vllm-bin` and `--sglang-python` are for an engine you
+installed yourself. For the agent to use yours rather than install its own,
+also set that engine's version to `""` under `engines` in `server.yaml`.
 
-So the node is not finished until:
-
-- the engines its catalog profiles name are installed (`vllm`, `sglang`,
-  `llama-server`), at the paths the agent configuration gives
-- the weights are on disk, at the `source` each model declares
-
-Both are deliberately outside this script. Engines are a platform decision with
-versions and drivers attached, and distributing weights is its own problem —
-hundreds of gigabytes that want a shared filesystem or a cache, not an scp from
-a laptop. llmgw will fetch weights for a model whose `source` is a URL, one
-copy per node, which covers the simple case; anything larger stays the
-operator's arrangement, and llmgw is content to find the files already there.
+Weights are the other thing a node needs. llmgw fetches them for a model whose
+`source` is a URL, one copy per node, which covers the simple case; anything
+larger — a shared filesystem, a cache — stays the operator's arrangement, and
+llmgw is content to find the files already there.
 
 The `sim` engine needs neither, which is what makes it useful for exercising
 everything else.
@@ -178,18 +151,24 @@ was cut short.
 
 ## Fabrics
 
-`--fabric` is how a node says which InfiniBand fabric it is attached to, and
-there is no way around stating it. OpenSM's default subnet prefix is `fe80::` on
-every fabric, so a matching prefix proves nothing and a wrong guess produces a
-replica that starts, passes health checks, and then hangs on its first
-all-reduce.
+`--fabric` is how a node says which InfiniBand fabric it is attached to. The
+name is checked against what the adapters report — every port on one fabric
+sees the same subnet managers — so two independent fabrics given one name are
+placed as two, and the cluster page says so; give each fabric its own name.
+The agent installs `infiniband-diags`, which that check needs, if it is missing.
+
+Before a node with a fabric joins, run `sudo llmgw doctor` on it: it checks the
+subnet manager, GPUDirect RDMA, the fabric manager and IMEX on NVLink machines,
+and the declared fabrics against the adapters. [fabrics.md](fabrics.md) has the
+rest.
 
 Omit it for a node with no fabric. Such a node is its own island: it can host
 single-node replicas and will never be given a multi-node one.
 
 Two adapters per fabric is the default; `--fabric ib-b:mlx5_2,mlx5_3` names them
 explicitly. An adapter must never appear under two fabrics — the agent refuses
-that configuration, because the consequence is the hang above.
+that configuration: a replica given adapters on two fabrics starts, passes its
+health checks, and hangs on its first all-reduce.
 
 ## Where the driver is
 

docs/onboarding.md

@@ -39,16 +39,40 @@ The landing page then links to it, and the rest is:
 
 ```
 llmgw users list --pending --store …
-ID                    EMAIL              NAME   STATE    PROJECT  SIGNED UP
-usr-k4mq…             ayse@example.com   Ayşe   pending  -        4m12s ago
+ID          EMAIL              STATE    ADMIN  MEMBER OF     SIGNED UP
+usr-k4mq…   ayse@example.com   pending  -      on their own  4m12s ago
 
-llmgw users approve usr-k4mq… --project default --by ilker --store …
+llmgw users approve ayse@example.com --by ilker --store …
+llmgw users approve ayse@example.com --project Acme/ml/training --store …   # or straight into a project
 ```
 
 That is the whole operator loop. From then on they sign in at `/login`, and
-`/profile` is where they create keys, see when each was last used, and revoke the
-ones they are done with. A key is shown once on the page that creates it and
-never again.
+their account page (`/profile`) is where they create keys, see when each was
+last used, revoke the ones they are done with, and see their own use over the
+last thirty days against every budget that applies to them. A key is shown once
+on the page that creates it and never again.
+
+### Connecting an editor
+
+The account page's **Connect your editor** makes a key for Continue, Codex or
+Cline and shows what the tool needs with that key, this gateway's address and a
+model already in it:
+
+- **Continue** — an assistant file of its own, `~/.continue/assistants/llmgw.yaml`,
+  so their `config.yaml` is not touched.
+- **Codex** — a provider and a profile in `~/.codex/config.toml`, run with
+  `codex --profile llmgw`; their default model stays as it was. Codex speaks the
+  Responses API, which the gateway translates at `/v1/responses`.
+- **Cline** — four settings to paste into its own screen, since it keeps them
+  in VS Code's secret storage where no file reaches.
+
+For Continue and Codex there is a one-line command,
+`curl -fsSL <gateway>/setup/<tool>?model=… | sh`. Its script is served without
+signing in and holds no secret — the key comes from the environment — and it
+keeps a copy of anything it replaces. Revoking the key disconnects the tool.
+
+How well an editor's agent mode works depends on the model: it needs one that
+calls tools well, which the coder models do and small general ones often do not.
 
 **This gateway sends no email.** There is no "your account was approved" message
 and no password reset, so tell people what to expect: they sign up, they tell
@@ -105,15 +129,70 @@ The tree is not fixed once made:
 llmgw org rename project Acme/platform/gateway api-gateway
 llmgw org move project Acme/platform/gateway --to Scania/ml
 llmgw org remove team Acme/old                 # only when nothing is in it
-llmgw users move ayse@example.com --project Acme/ml/research
 llmgw users delete ayse@example.com --yes      # signs out, revokes their keys
-llmgw keys list --all                          # every key, its project and owner
+llmgw keys list --all                          # every key, where it spends, and whose
 ```
 
-Moving a person takes the keys they made with them, so they stop spending the
-old project's budget; keys an operator issued in the project stay with it. A
-project that ever had a key cannot be removed, because usage history refers to
-it — rename it instead.
+A project that ever had a key cannot be removed, because usage history refers
+to it — rename it instead.
+
+### Where people belong
+
+A person can be a member of any number of organizations, teams and projects, or
+of none:
+
+```
+llmgw users join ayse@example.com --team Acme/ml          # every project in ml
+llmgw users join ayse@example.com --project Scania/research/vision
+llmgw users leave ayse@example.com --project Acme/ml/training
+```
+
+A member of a team or organization may use every project in it. Leaving revokes
+the keys the person made in projects they can no longer reach; a project they
+still reach through another membership keeps them.
+
+Someone who is a member of nothing still uses the gateway, on their own:
+personal keys, and the chat page, within their personal limits. When they make
+a key on `/profile` they choose what it spends against — one of their projects,
+or "Personal". An operator can do the same:
+
+```
+llmgw keys create laptop --for ayse@example.com                      # personal
+llmgw keys create ci --for ayse@example.com --project Acme/ml/training
+```
+
+Personal use can be limited by default, or turned off so people use the gateway
+only through their projects:
+
+```yaml
+auth:
+  personal:
+    rpm: 20
+    tpm: 50000
+    budget: 10        # whole units
+    period: month
+    # disabled: true
+```
+
+A personal key has no project, so it keeps no conversations or agent notes;
+those are kept per project.
+
+### Organization managers
+
+A member of an organization can be made its manager:
+
+```
+llmgw users join boss@acme.com --org Acme --role manager
+```
+
+or with the star on their organization membership on `/admin/users`. A manager
+gets an **Organization** link and a page at `/org`: usage over time by team,
+project or person, the teams and projects with their limits and spend, and every
+member. Only what was spent in that organization's projects is on it; what its
+people do elsewhere is not. The page is read-only — a manager is not an
+administrator of the cluster, and limits and memberships stay with those who
+are. An administrator following `/org` lands on the same organization's panel
+on the people page instead, where all of that can be changed.
 
 `llmgw keys init` still works for provisioning scripts and is idempotent, but it
 will no longer move a team or project that already exists under another parent;
@@ -181,7 +260,7 @@ Allowlists compose by intersection: a key cannot add a model its team withheld.
 A model's canonical name and any of its aliases name the same thing, so either
 form works.
 
-Budgets are given in whole units, as the quotas page shows them, and what a unit
+Budgets are given in whole units, as the Limits tab shows them, and what a unit
 means is yours to decide. Self-hosted models have no per-token price, so it can
 be GPU-seconds, a notional rate, or one unit per thousand tokens everywhere —
 a price of 1 000 000 in `auth.pricing`, which is in micro-units — and a budget
@@ -195,6 +274,20 @@ llmgw keys limits --store … --scope project --id Acme/platform/gateway \
   --budget 50 --period month              # 50 units a month
 ```
 
+Two more kinds of limit follow people rather than the tree:
+
+```
+# A person, everywhere: every key they hold, every project, the chat page
+llmgw keys limits --scope user --id ayse@example.com --budget 25
+
+# Each member of a project separately — a fair share, not one shared pool
+llmgw keys limits --scope project --id Acme/ml/training --per-member --rpm 20
+```
+
+A request is held to all of them at once: the key's own, its project's, team's
+and organization's, each per-member limit on those for its owner, and the
+owner's own. The strictest wins, and each is counted in its own pool.
+
 ## The landing page
 
 Someone who reaches the gateway without a key sees the landing page. Tell it who
@@ -314,11 +407,22 @@ page exists.
 ## Managing accounts from the browser
 
 `/admin/users` is the tree: organizations, teams and projects, each project
-opening onto its people and its keys. From there an admin approves, moves,
-suspends and removes people, revokes keys, and adds, renames, moves and removes
-organizations, teams and projects. A searchable list of every account is at the
-bottom. The page works as plain forms; with scripts on, a change updates it in
-place and the projects you had open stay open.
+opening onto its members and its keys. From there an admin approves people
+(into a project or on their own), adds them to and takes them out of
+organizations, teams and projects, suspends and removes them, revokes keys, and
+adds, renames, moves and removes organizations, teams and projects. A
+searchable list of every account is at the bottom. Choosing any organization,
+team, project or person opens a panel beside the tree with its limits, its
+capacity quota and its use over the last week, and its limits can be changed
+there.
+
+The same page has two more tabs, each at the address it had as a page of its
+own: **Limits** (`/admin/quotas`) is every limit in the tree at once — per
+branch, per member of any part of it, and per person — with the capacity quotas;
+**Usage** (`/admin/usage`) is the cluster's use over time by organization, team,
+project or person, and the GPU time each owner held. The page works as plain
+forms; with scripts on, a change updates it in place and the projects you had
+open stay open.
 
 Three things are deliberately not there:
 

docs/quick-install.md

@@ -0,0 +1,202 @@
+# Quick install
+
+From empty servers to a model answering requests, in six steps. This page
+has only the commands and what to check after each; [install.md](install.md)
+explains every choice.
+
+## What you need
+
+| | |
+|---|---|
+| **One control plane machine** | Linux, x86-64 or ARM64. Debian 12, Ubuntu 22.04+ or RHEL 10. No GPU needed; 2 GB RAM, 20 GB disk. |
+| **GPU nodes** (optional to start) | The same Linux, with the NVIDIA driver installed. vLLM needs driver 580 or newer and a Turing GPU or newer (not V100). |
+| **root** | on the control plane, and `sudo` without a password on every node. |
+| **SSH** | from the control plane to each node, as a user with passwordless `sudo` there. |
+| **Network** | nodes reach the control plane on 7443 and 8081; users reach it on 8080. The machines need internet access to download engines and models. |
+
+You do not install llama.cpp, vLLM, Python or CUDA. The nodes do that
+themselves once they join.
+
+## 1. Put llmgw on the control plane
+
+```
+VERSION=$(curl -fsSL https://gitea.mynodes.xyz/api/v1/repos/ilker/llmgw/releases/latest \
+  | grep -o '"tag_name":"[^"]*"' | cut -d'"' -f4)
+U=https://gitea.mynodes.xyz/ilker/llmgw/releases/download/$VERSION
+curl -fsSLO $U/llmgw-$VERSION-linux-amd64          # -linux-arm64 on ARM
+curl -fsSL $U/checksums.txt | sha256sum -c --ignore-missing
+sudo install -m0755 llmgw-$VERSION-linux-amd64 /usr/local/bin/llmgw
+llmgw --version
+```
+
+`sha256sum` must print `OK`. If it does not, the download is damaged; do not
+install it.
+
+## 2. Set up the control plane
+
+```
+sudo llmgw server init
+```
+
+It asks for your email and a few choices, then sets up everything on this
+machine, in a few minutes. In a script, give them as flags instead:
+`sudo llmgw server init --admin you@example.com`.
+
+- the service, its certificates and its configuration;
+- your administrator account. **The password is printed once. Copy it now.**
+- a model mirror, which downloads model files once for the whole cluster;
+- this machine as a first node, and a small model (Qwen2.5 0.5B) on it.
+
+If nodes will reach this machine by a name or address it does not know itself
+(a DNS name, a NAT address), add `--host` for each:
+
+```
+sudo llmgw server init --admin you@example.com --host llm.example.com
+```
+
+It is safe to run again. It keeps what exists and finishes what is missing.
+
+**The mirror on its own machine**, when it wants a larger disk than the
+control plane has: first on that machine, with llmgw installed as in step 1,
+
+```
+sudo llmgw mirror init --control-plane <control plane address>
+```
+
+then on the control plane `sudo llmgw server init --mirror http://<mirror
+address>:8082`. The mirror serves model files to every node, and takes
+orders only from the control plane. Untick "This machine as a node" in the
+form (or add `--no-node`) to keep models off the control plane; the starter
+model then starts on the first node you add.
+
+**Check:** open `http://<control-plane>:8080/chat`, sign in with the printed
+password, and ask something. The small model answers. Change the password on
+the account page.
+
+## 3. Add the GPU nodes
+
+On the control plane:
+
+```
+sudo llmgw node add ubuntu@10.0.0.21
+```
+
+Several at once: `sudo llmgw node add ubuntu@10.0.0.21 ubuntu@10.0.0.22 ubuntu@10.0.0.23`.
+
+Replace `ubuntu` with the user you log in to the node as, and `10.0.0.21` with
+its address. That user needs `sudo` without a password on the node.
+
+It logs in over SSH and, while you watch:
+
+1. installs llmgw on the node, the release for its processor, checksum checked;
+2. runs `llmgw doctor` there, and stops if the node has a problem. Each problem
+   comes with the command that fixes it. Fix it and run the same command again;
+3. gives the node its certificate and configuration, and starts its agent;
+4. waits for the node to join and install llama.cpp.
+
+The node's name in the cluster is its hostname. vLLM installs by itself after
+that, in a few more minutes on a GPU node (about 8 GB). The cluster page
+(`/admin`) shows its progress, or why the node cannot run it.
+
+**InfiniBand.** When the node's adapters are up, `node add` shows them grouped
+by the fabric they are on and asks for each fabric's name. Give every node on
+the same fabric the same name. In a script, give it instead:
+
+```
+sudo llmgw node add ubuntu@10.0.0.21 --fabric ib-a:mlx5_0,mlx5_1,mlx5_2,mlx5_3
+```
+
+Once two nodes on a fabric are in, check they reach each other at the link's
+rate before placing anything across them:
+
+```
+llmgw ctl fabric-test gpu01 gpu02
+```
+
+**If it cannot log in.** `sudo` runs SSH as root, with root's keys. To use your
+own keys instead, log in to the control plane with `ssh -A` and keep your
+agent through sudo:
+
+```
+sudo --preserve-env=SSH_AUTH_SOCK llmgw node add ubuntu@10.0.0.21
+```
+
+**Check:** `llmgw ctl nodes` lists the node. Running `node add` again on the
+same node is safe: it renews its certificate and configuration, and keeps what
+runs there.
+
+A machine retired for good is removed with `llmgw node remove <name>` once its
+agent is stopped.
+
+## 4. Add a model
+
+Open `/admin/mirror` and choose **Explore models**. It lists what is popular on
+Hugging Face, showing only what your nodes can run.
+
+1. Click a model to read its card.
+2. Pick one of its files, usually a `q4_k_m` for a first try, and click
+   **Fetch**. It is downloaded into the mirror once, for every node.
+3. When the download finishes, click **serve** on its row in the table above.
+4. In the form, choose how it runs: **always on**, or **on demand**, which
+   loads it on the first request and stops it after 15 minutes unused. Click
+   **Add to the catalog**.
+
+The model is placed on a node that fits it, and its status on `/admin` turns
+`ready`.
+
+**Check:** the model appears in the list on `/chat`. Ask it something.
+
+## 5. Give people access
+
+**A key for a program:** each person makes their own on the account page
+(`/profile`). The key is shown once.
+
+```
+curl http://<control-plane>:8080/v1/chat/completions \
+  -H "Authorization: Bearer $LLMGW_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{"model": "<model-name>", "messages": [{"role": "user", "content": "hello"}]}'
+```
+
+Any OpenAI client works. Only the base URL changes:
+`http://<control-plane>:8080/v1`.
+
+**An editor:** the account page connects Continue, Codex or Cline. Pick the
+tool and a model, and it gives a one-line command that writes the
+configuration.
+
+**Other people's accounts:** they sign up at `/signup`, and you approve them on
+the people page (`/admin/users`).
+
+## 6. Put it behind HTTPS
+
+Port 8080 is plain HTTP. Before users outside a trusted network use it, put a
+reverse proxy in front with a real certificate. With Caddy, which gets the
+certificate by itself:
+
+```
+sudo apt-get install -y caddy          # Debian and Ubuntu
+curl -fsSL $U/Caddyfile | sed 's/llm.example.com/YOUR.DOMAIN/' | sudo tee /etc/caddy/Caddyfile
+sudo systemctl reload caddy
+```
+
+Keep 7443 (nodes) and 8081 (mirror) on the internal network. They never go
+through the proxy. There is an nginx configuration too:
+[deploy/proxy/](../deploy/proxy/).
+
+## When something does not work
+
+| What you see | What to do |
+|---|---|
+| `node add` stops at `join` | `sudo journalctl -u llmgw-agent -n 50` on the node. Usually it cannot reach port 7443 on the control plane. |
+| `node add` says the address is not in the certificate | Run `sudo llmgw server init --host <that address>`, then `node add` again. |
+| A node says it cannot run vLLM | The reason is on `/admin`: the driver is older than 580, the GPU is older than Turing, or there is less than 25 GB free. |
+| A model shows `not placed` | `llmgw ctl explain-placement <model>` names each node and why it was not used. |
+| A reply stops after 60 seconds behind nginx | Use the nginx configuration from the release. Its timeouts are set for slow models. |
+| Anything else about a node | `sudo llmgw doctor` on it. `--json` writes everything it read, to send to whoever helps you. |
+
+## Next
+
+- [install.md](install.md): the same steps with every option explained.
+- [fabrics.md](fabrics.md): InfiniBand and NVLink.
+- [onboarding.md](onboarding.md): accounts, projects, budgets and keys.

docs/quickstart.md

@@ -1,8 +1,8 @@
 # Quickstart: one machine, one model, ten minutes
 
-Written for someone who wants to see llmgw answer a request on a single
-workstation before reading anything else. No systemd, no `sudo`. For a real
-cluster, see [install.md](install.md).
+Written for a developer who wants to see llmgw answer a request on a single
+workstation, built from source, with no systemd and no `sudo`. **To install
+llmgw on servers, read [quick-install.md](quick-install.md) instead.**
 
 What you end up with: an OpenAI-compatible endpoint on `localhost:8080` in front
 of a real model served by llama.cpp.

go.mod

@@ -6,15 +6,24 @@ require (
 	connectrpc.com/connect v1.21.0
 	github.com/NVIDIA/go-nvml v0.13.4-0
 	github.com/cespare/xxhash/v2 v2.3.0
+	github.com/charmbracelet/bubbletea v1.3.10
+	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/jackc/pgx/v5 v5.11.0
 	github.com/knadh/koanf/parsers/yaml v1.1.1
 	github.com/knadh/koanf/providers/env/v2 v2.0.1
 	github.com/knadh/koanf/providers/file v1.2.1
 	github.com/knadh/koanf/v2 v2.3.6
+	github.com/mattn/go-isatty v0.0.24
 	github.com/prometheus/client_golang v1.24.1
 	github.com/prometheus/client_model v0.6.3
 	github.com/prometheus/common v0.71.0
 	github.com/spf13/cobra v1.10.2
+	github.com/yuin/goldmark v1.8.6
+	go.opentelemetry.io/otel v1.46.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0
+	go.opentelemetry.io/otel/sdk v1.46.0
+	go.opentelemetry.io/otel/trace v1.46.0
+	go.yaml.in/yaml/v3 v3.0.5
 	golang.org/x/crypto v0.57.0
 	golang.org/x/net v0.59.0
 	golang.org/x/sync v0.23.0
@@ -23,27 +32,54 @@ require (
 )
 
 require (
+	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
+	github.com/charmbracelet/colorprofile v0.4.1 // indirect
+	github.com/charmbracelet/x/ansi v0.11.6 // indirect
+	github.com/charmbracelet/x/cellbuf v0.0.15 // indirect
+	github.com/charmbracelet/x/term v0.2.2 // indirect
+	github.com/clipperhouse/displaywidth v0.9.0 // indirect
+	github.com/clipperhouse/stringish v0.1.1 // indirect
+	github.com/clipperhouse/uax29/v2 v2.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
-	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/go-logr/logr v1.4.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/knadh/koanf/maps v0.1.2 // indirect
-	github.com/mattn/go-isatty v0.0.24 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
+	github.com/mattn/go-localereader v0.0.1 // indirect
+	github.com/mattn/go-runewidth v0.0.19 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
+	github.com/muesli/cancelreader v0.2.2 // indirect
+	github.com/muesli/termenv v0.16.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/ncruces/go-strftime v1.0.0 // indirect
 	github.com/prometheus/procfs v0.21.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
-	go.yaml.in/yaml/v3 v3.0.5 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect
+	go.opentelemetry.io/otel/metric v1.46.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
 	golang.org/x/sys v0.48.0 // indirect
 	golang.org/x/text v0.42.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect
+	google.golang.org/grpc v1.83.1 // indirect
 	modernc.org/libc v1.75.7 // indirect
 	modernc.org/mathutil v1.7.1 // indirect
 	modernc.org/memory v1.12.1 // indirect

go.sum

@@ -2,24 +2,57 @@ connectrpc.com/connect v1.21.0 h1:LhqSJt7jHf5NJBo9Jq/t/9FjcYAideif0mg+qe2jCUs=
 connectrpc.com/connect v1.21.0/go.mod h1:A2ygJrukXwWy32vkCAAHNVguZrqZ+jeZ9rGRnGR4dN4=
 github.com/NVIDIA/go-nvml v0.13.4-0 h1:o3jp9u2x1R9ShFE3v+Aesp55XOSIQFMJz/VGNUcJaNE=
 github.com/NVIDIA/go-nvml v0.13.4-0/go.mod h1:id63qwpoDWpFXwnwM6psDCSqW4BmNu6mWpr4YeQtPGo=
+github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
+github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
+github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
+github.com/charmbracelet/colorprofile v0.4.1 h1:a1lO03qTrSIRaK8c3JRxJDZOvhvIeSco3ej+ngLk1kk=
+github.com/charmbracelet/colorprofile v0.4.1/go.mod h1:U1d9Dljmdf9DLegaJ0nGZNJvoXAhayhmidOdcBwAvKk=
+github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
+github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
+github.com/charmbracelet/x/ansi v0.11.6 h1:GhV21SiDz/45W9AnV2R61xZMRri5NlLnl6CVF7ihZW8=
+github.com/charmbracelet/x/ansi v0.11.6/go.mod h1:2JNYLgQUsyqaiLovhU2Rv/pb8r6ydXKS3NIttu3VGZQ=
+github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI=
+github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q=
+github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
+github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
+github.com/clipperhouse/displaywidth v0.9.0 h1:Qb4KOhYwRiN3viMv1v/3cTBlz3AcAZX3+y9OLhMtAtA=
+github.com/clipperhouse/displaywidth v0.9.0/go.mod h1:aCAAqTlh4GIVkhQnJpbL0T/WfcrJXHcj8C0yjYcjOZA=
+github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
+github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
+github.com/clipperhouse/uax29/v2 v2.5.0 h1:x7T0T4eTHDONxFJsL94uKNKPHrclyFI0lm7+w94cO8U=
+github.com/clipperhouse/uax29/v2 v2.5.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
+github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
-github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
-github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
+github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
+github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
 github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@@ -46,12 +79,24 @@ github.com/knadh/koanf/v2 v2.3.6 h1:JoQPSJmvS4aP0xNc8xMDr5tcrkSEInL23/Il7pITAKo=
 github.com/knadh/koanf/v2 v2.3.6/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
+github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
 github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
+github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
+github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
+github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
+github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
+github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
+github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
+github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
+github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
+github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
@@ -67,6 +112,8 @@ github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+
 github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
 github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
@@ -77,6 +124,28 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
 github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
+github.com/yuin/goldmark v1.8.6 h1:d0VcaP1sx9GkFVkoW+KtggpGi2KZ965i14b0+bDQST4=
+github.com/yuin/goldmark v1.8.6/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
+go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 h1:KrC1YrQeSt46ITMWAbgQx1M1eV1/1TKzttrBzymPmss=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0/go.mod h1:zDSEzoEqsOrgBeGvH66KRgxh90VonFyJqBHA0Pk3+rM=
+go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8=
+go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o=
+go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI=
+go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM=
+go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE=
+go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4=
+go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c=
+go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI=
+go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
+go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
@@ -86,18 +155,31 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
 go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
 golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
 golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
 golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
 golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
 golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
 golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
 golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
 golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
 golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
+golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
 golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
 golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
 golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
 golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
+google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI=
+google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 h1:cYNAzI2sUwhmCcoj9TxvihSrqsxt6uIkj3rDRhSDmW4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA=
+google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y=
+google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
 google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
 google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/account/account_test.go

@@ -99,17 +99,21 @@ func TestApprovalIsWhatMakesAnAccountWork(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if got.ProjectID != "proj" {
-		t.Errorf("the approved account landed in project %q", got.ProjectID)
+	if ok, err := st.CanUseProject(ctx, got.ID, "proj"); err != nil || !ok {
+		t.Errorf("the approved account is not a member of the project it was approved into: %v", err)
 	}
 }
 
-func TestApprovalNeedsAProject(t *testing.T) {
-	// An approved account with nowhere to put its keys passes every check and
-	// then fails at the first key it tries to make.
-	_, st := newStoreAndUser(t)
-	if err := st.ApproveUser(context.Background(), "usr-1", "", "ilker", time.Now()); err == nil {
-		t.Error("an account was approved into no project")
+func TestApprovalNeedNotNameAProject(t *testing.T) {
+	// A person need not belong to anything to use the gateway: approved into
+	// nothing, they work on their own within their personal limits.
+	svc, st := newStoreAndUser(t)
+	ctx := context.Background()
+	if err := st.ApproveUser(ctx, "usr-1", "", "ilker", time.Now()); err != nil {
+		t.Fatalf("approving with no project: %v", err)
+	}
+	if _, _, err := svc.SignIn(ctx, "ayse@example.com", goodPassword); err != nil {
+		t.Errorf("an account approved into nothing cannot sign in: %v", err)
 	}
 }
 

internal/account/password.go

@@ -15,6 +15,7 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"runtime"
 	"strings"
 
 	"golang.org/x/crypto/argon2"
@@ -56,7 +57,7 @@ func HashPassword(password string) (string, error) {
 	if _, err := rand.Read(salt); err != nil {
 		return "", err
 	}
-	sum := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen)
+	sum := idKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen)
 
 	b64 := base64.RawStdEncoding.EncodeToString
 	return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s",
@@ -93,7 +94,7 @@ func VerifyPassword(password, encoded string) (bool, error) {
 		return false, ErrBadHash
 	}
 
-	got := argon2.IDKey([]byte(password), salt, time, memory, threads, uint32(len(want)))
+	got := idKey([]byte(password), salt, time, memory, threads, uint32(len(want)))
 	return subtle.ConstantTimeCompare(got, want) == 1, nil
 }
 
@@ -110,3 +111,16 @@ func init() {
 	}
 	dummyHash = h
 }
+
+// argonSlots bounds how many hashes run at once. Each takes 64 MiB, and the
+// login and signup forms are on the client-facing port: without a bound, a
+// few hundred parallel attempts at random addresses asked for gigabytes at
+// once, and the per-address throttle does not see them — each address is new.
+// Beyond the bound, attempts wait their turn instead.
+var argonSlots = make(chan struct{}, max(2, runtime.NumCPU()/2))
+
+func idKey(password, salt []byte, time, memory uint32, threads uint8, keyLen uint32) []byte {
+	argonSlots <- struct{}{}
+	defer func() { <-argonSlots }()
+	return argon2.IDKey(password, salt, time, memory, threads, keyLen)
+}

internal/account/service.go

@@ -230,6 +230,22 @@ func (s *Service) allowAttempt(email string) bool {
 	defer s.mu.Unlock()
 
 	w, ok := s.attempts[email]
+	if !ok && len(s.attempts) >= maxTrackedAddresses {
+		// Full of addresses nobody has signed in as for a while: forget the
+		// ones whose window has passed. Only expiring, never evicting early —
+		// dropping a live window is how an attacker spraying random addresses
+		// would buy unlimited guesses at the one they care about.
+		for k, v := range s.attempts {
+			if now.Sub(v.since) > s.cfg.AttemptWindow {
+				delete(s.attempts, k)
+			}
+		}
+		if len(s.attempts) >= maxTrackedAddresses {
+			// Still full, so somebody is spraying addresses faster than they
+			// expire. New ones wait; those already tracked are unaffected.
+			return false
+		}
+	}
 	if !ok || now.Sub(w.since) > s.cfg.AttemptWindow {
 		s.attempts[email] = &attemptWindow{count: 1, since: now}
 		return true
@@ -238,6 +254,10 @@ func (s *Service) allowAttempt(email string) bool {
 	return w.count <= s.cfg.MaxAttempts
 }
 
+// maxTrackedAddresses bounds the throttle's memory. Unbounded, it grew by one
+// entry per address tried and shrank only on a successful sign-in.
+const maxTrackedAddresses = 50000
+
 func (s *Service) clearAttempts(email string) {
 	s.mu.Lock()
 	delete(s.attempts, email)

internal/account/throttle_bound_test.go

@@ -0,0 +1,33 @@
+package account
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestSprayingAddressesCannotGrowTheThrottleOrEscapeIt(t *testing.T) {
+	now := time.Unix(0, 0)
+	s := &Service{cfg: DefaultConfig(), attempts: map[string]*attemptWindow{}, now: func() time.Time { return now }}
+
+	// The target is being guessed at.
+	for i := 0; i < s.cfg.MaxAttempts; i++ {
+		s.allowAttempt("victim@example.com")
+	}
+	// Then the table is flooded with made-up addresses.
+	for i := 0; i < maxTrackedAddresses+100; i++ {
+		s.allowAttempt(fmt.Sprintf("x%d@example.com", i))
+	}
+	if n := len(s.attempts); n > maxTrackedAddresses {
+		t.Errorf("the table grew to %d", n)
+	}
+	if s.allowAttempt("victim@example.com") {
+		t.Error("flooding the table reset the target's throttle")
+	}
+
+	// Once the window passes, everything expires and sign-ins work again.
+	now = now.Add(s.cfg.AttemptWindow + time.Second)
+	if !s.allowAttempt("new@example.com") {
+		t.Error("a new address was refused after the flood expired")
+	}
+}

internal/accounting/accounting.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"strings"
 	"time"
 
 	"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
@@ -79,14 +80,27 @@ func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period s
 	// totals would bring the control plane back up believing the team had spent
 	// nothing — which is a budget that resets on every restart, for exactly the
 	// scopes where it matters most.
-	spend := make(map[string]int64, len(rows)*4)
+	// And every pool a person is in: their own, and each per-member limit of
+	// the project, team and organization they spent it in.
+	spend := make(map[string]int64, len(rows)*8)
 	for _, r := range rows {
-		for _, o := range []store.LimitOwner{
+		owners := []store.LimitOwner{
 			{Scope: store.ScopeKey, ID: r.KeyID},
 			{Scope: store.ScopeProject, ID: r.ProjectID},
 			{Scope: store.ScopeTeam, ID: r.TeamID},
 			{Scope: store.ScopeOrg, ID: r.OrgID},
-		} {
+			{Scope: store.ScopeUser, ID: r.UserID},
+		}
+		if r.UserID != "" && r.ProjectID != "" {
+			owners = append(owners,
+				store.MemberOwner(store.ScopeProjectMember, r.ProjectID, r.UserID),
+				store.MemberOwner(store.ScopeTeamMember, r.TeamID, r.UserID),
+				store.MemberOwner(store.ScopeOrgMember, r.OrgID, r.UserID))
+		}
+		for _, o := range owners {
+			if o.ID == "" || strings.HasPrefix(o.ID, "/") {
+				continue
+			}
 			if k := o.CounterKey(); k != "" {
 				spend[k] += r.Micros
 			}
@@ -302,6 +316,17 @@ func (a *Accountant) Record(res Reservation, u store.Usage) {
 	}
 }
 
+// RecordRefused queues the row of a request a limit refused: nothing is
+// charged — it used nothing — but it is counted, so a team running into its
+// limits shows as one rather than as a team that went quiet.
+func (a *Accountant) RecordRefused(u store.Usage) {
+	if u.KeyID == "" || a.recorder == nil {
+		return
+	}
+	u.PromptTokens, u.CompletionTokens, u.CostMicros = 0, 0, 0
+	a.recorder.Record(u)
+}
+
 // Status returns what a key has used, for headers and `ctl`.
 func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) {
 	if p.KeyID == "" {
@@ -369,3 +394,39 @@ func keyCounter(keyID string) string {
 // they are asking is whether the budgets they are about to display mean
 // anything, and the accountant is what would enforce them.
 func (a *Accountant) AnythingPriced() bool { return a.pricing.AnythingPriced() }
+
+// Headroom is the tightest of a principal's request and token limits: how
+// much is left of each, and when the minute it is counted over rolls on.
+type Headroom struct {
+	RequestLimit, RequestsLeft int
+	TokenLimit, TokensLeft     int
+	Reset                      time.Duration
+}
+
+// Headroom reports what a principal has left, after whatever Check just
+// charged. Of several levels the one with least left is the one that will
+// refuse first, so that is the one reported. A limit of 0 means none is set.
+func (a *Accountant) Headroom(p auth.Principal) Headroom {
+	var h Headroom
+	firstReq, firstTok := true, true
+	for _, l := range levels(p.Limits, p.KeyID) {
+		if l.RPM != nil && *l.RPM > 0 {
+			left := max(0, *l.RPM-int(a.requests.Rate(l.Counter)))
+			if firstReq || left < h.RequestsLeft {
+				h.RequestLimit, h.RequestsLeft, firstReq = *l.RPM, left, false
+			}
+		}
+		if l.TPM != nil && *l.TPM > 0 {
+			left := max(0, *l.TPM-int(a.tokens.Rate(l.Counter)))
+			if firstTok || left < h.TokensLeft {
+				h.TokenLimit, h.TokensLeft, firstTok = *l.TPM, left, false
+			}
+		}
+	}
+	// The windows slide, so room comes back continuously; the next minute
+	// boundary is when a whole window's worth has moved on, and is the figure
+	// a client pacing itself can act on.
+	now := time.Now()
+	h.Reset = now.Truncate(windowLen).Add(windowLen).Sub(now)
+	return h
+}

internal/accounting/accounting_test.go

@@ -630,3 +630,31 @@ func TestAnEmptyPriceTableMeansNoBudgetCanEverBind(t *testing.T) {
 		t.Error("no price table at all is not priced")
 	}
 }
+
+func TestHeadroomIsTheTightestLimitAfterThisRequest(t *testing.T) {
+	// A project allowing 100 a minute under a team allowing 3: the team is
+	// what will refuse, so the team is what a client is told about.
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	p := principal("k1", func(e *store.Effective) {
+		e.Chain = []store.Limits{
+			{Scope: store.ScopeProject, ScopeID: "proj", RPM: intp(100), TPM: intp(50000)},
+			{Scope: store.ScopeTeam, ScopeID: "team", RPM: intp(3)},
+		}
+	})
+	if _, err := a.Check(p, "m", 10); err != nil {
+		t.Fatal(err)
+	}
+	h := a.Headroom(p)
+	if h.RequestLimit != 3 || h.RequestsLeft != 2 {
+		t.Errorf("requests %d left of %d; expected 2 of the team's 3", h.RequestsLeft, h.RequestLimit)
+	}
+	if h.TokenLimit != 50000 || h.TokensLeft != 50000-10 {
+		t.Errorf("tokens %d left of %d", h.TokensLeft, h.TokenLimit)
+	}
+	if h.Reset <= 0 || h.Reset > time.Minute {
+		t.Errorf("reset %v", h.Reset)
+	}
+	if none := a.Headroom(principal("k2", nil)); none.RequestLimit != 0 || none.TokenLimit != 0 {
+		t.Errorf("a key with no limits advertised some: %+v", none)
+	}
+}

internal/accounting/people_test.go

@@ -0,0 +1,126 @@
+package accounting
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+// peopleStore has Acme/ml/training with two members, ayse and deniz.
+func peopleStore(t *testing.T) (*store.DB, string) {
+	t.Helper()
+	ctx := context.Background()
+	s, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "p.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = s.Close() })
+	org, _ := s.CreateOrg(ctx, "Acme")
+	team, _ := s.CreateTeam(ctx, org, "ml")
+	proj, _ := s.CreateProject(ctx, team, "training")
+	for _, id := range []string{"ayse", "deniz"} {
+		if err := s.CreateUser(ctx, store.User{ID: id, Email: id + "@x", Name: id,
+			PasswordHash: "h", State: store.UserPending, CreatedAt: time.Now()}); err != nil {
+			t.Fatal(err)
+		}
+		if err := s.ApproveUser(ctx, id, proj, "test", time.Now()); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return s, proj
+}
+
+// keyFor makes a key for a person and resolves it the way authentication does.
+func keyFor(t *testing.T, s *store.DB, id, user, project string) auth.Principal {
+	t.Helper()
+	ctx := context.Background()
+	_, k, _ := store.NewKey(id, id, project)
+	if err := s.CreateUserKey(ctx, user, k); err != nil {
+		t.Fatal(err)
+	}
+	e, err := s.Resolve(ctx, id)
+	if err != nil {
+		t.Fatalf("resolving %s: %v", id, err)
+	}
+	return auth.Principal{KeyID: id, UserID: e.UserID, ProjectID: e.ProjectID, Limits: e}
+}
+
+func TestAPersonsLimitIsOnePoolAcrossTheirKeys(t *testing.T) {
+	s, proj := peopleStore(t)
+	if err := s.PutLimits(context.Background(),
+		store.Limits{Scope: store.ScopeUser, ScopeID: "ayse", RPM: intp(2)}); err != nil {
+		t.Fatal(err)
+	}
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	personal := keyFor(t, s, "k-personal", "ayse", "")
+	inProject := keyFor(t, s, "k-project", "ayse", proj)
+
+	// Two requests, one on each key, use up the person's two a minute.
+	for _, p := range []auth.Principal{personal, inProject} {
+		if _, err := a.Check(p, "free", 1); err != nil {
+			t.Fatalf("%s: %v", p.KeyID, err)
+		}
+	}
+	if _, err := a.Check(personal, "free", 1); !errors.Is(err, ErrRateLimited) {
+		t.Errorf("a third request across her keys was allowed: %v", err)
+	}
+	// Deniz is somebody else.
+	if _, err := a.Check(keyFor(t, s, "k-deniz", "deniz", proj), "free", 1); err != nil {
+		t.Errorf("another person was limited by hers: %v", err)
+	}
+}
+
+func TestAPerMemberLimitCountsEachMemberApart(t *testing.T) {
+	s, proj := peopleStore(t)
+	if err := s.PutLimits(context.Background(),
+		store.Limits{Scope: store.ScopeProjectMember, ScopeID: proj, RPM: intp(1)}); err != nil {
+		t.Fatal(err)
+	}
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	ayse := keyFor(t, s, "k-a", "ayse", proj)
+	deniz := keyFor(t, s, "k-d", "deniz", proj)
+
+	if _, err := a.Check(ayse, "free", 1); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := a.Check(ayse, "free", 1); !errors.Is(err, ErrRateLimited) {
+		t.Errorf("her second request passed a one-a-minute per-member limit: %v", err)
+	}
+	// Her share is hers: Deniz still has his.
+	if _, err := a.Check(deniz, "free", 1); err != nil {
+		t.Errorf("one member's use counted against another: %v", err)
+	}
+	// And outside the project it does not apply at all.
+	if _, err := a.Check(keyFor(t, s, "k-a-own", "ayse", ""), "free", 1); err != nil {
+		t.Errorf("a project's per-member limit held her personal key: %v", err)
+	}
+}
+
+func TestAPersonsBudgetSurvivesARestart(t *testing.T) {
+	s, _ := peopleStore(t)
+	ctx := context.Background()
+	if err := s.PutLimits(ctx, store.Limits{Scope: store.ScopeUser, ScopeID: "ayse",
+		Budget: i64p(1000), BudgetPeriod: store.PeriodMonth}); err != nil {
+		t.Fatal(err)
+	}
+	// Spent before the restart, personally and on the chat page.
+	if err := s.WriteUsage(ctx, []store.Usage{
+		{At: time.Now(), KeyID: "k-old", UserID: "ayse", CostMicros: 600, Model: "paid", ServedModel: "paid"},
+		{At: time.Now(), KeyID: "session:ayse", UserID: "ayse", CostMicros: 500, Model: "paid", ServedModel: "paid"},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	if err := a.RestoreBudgets(ctx, s, store.PeriodMonth); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := a.Check(keyFor(t, s, "k-new", "ayse", ""), "paid", 1); !errors.Is(err, ErrBudgetSpent) {
+		t.Errorf("a restart forgot what she had spent: %v", err)
+	}
+}

internal/agent/agent.go

@@ -14,11 +14,10 @@ import (
 	"math/rand/v2"
 	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
+	"runtime"
 	"slices"
-	"strings"
 	"sync"
 	"time"
 
@@ -32,275 +31,9 @@ import (
 	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
-	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
 )
 
-// Config configures the agent.
-type Config struct {
-	// NodeID defaults to the hostname. It must equal the common name of the
-	// agent's certificate, so set it only where the two differ.
-	NodeID string `koanf:"node_id"`
-	// Server is the control plane, and Servers is the same thing when there is
-	// more than one. Both are accepted: every configuration written before
-	// there could be a second control plane names one address, and none of
-	// them should have to change.
-	//
-	// The order is tried in order, and a refusal moves to the next. Only the
-	// leader accepts an agent, so a standby is skipped in the time it takes to
-	// be told so — which is what makes failover a reconnect rather than an
-	// edit to every node's configuration.
-	Server  string   `koanf:"server"`
-	Servers []string `koanf:"servers"`
-	// Hostname defaults to the machine's.
-	Hostname string `koanf:"hostname"`
-	// MgmtAddr is the address engines are reached on. It defaults to this
-	// machine's address on the route to the control plane, which is the one
-	// the control plane and the router can reach.
-	MgmtAddr string `koanf:"mgmt_addr"`
-	// Labels are free-form facts about a node for a profile's node_selector.
-	// Nothing needs them: memory, GPUs, engines and disk are measured, and a
-	// label restating one of those goes stale the day the machine changes.
-	Labels map[string]string `koanf:"labels"`
-
-	// MgmtInterface is the ethernet interface NCCL bootstraps over. It
-	// defaults to the interface holding MgmtAddr, then eth0.
-	MgmtInterface string `koanf:"mgmt_interface"`
-	// Fabrics labels this node's HCAs. It is explicit on purpose: OpenSM's
-	// default subnet prefix is the same on every fabric, so nothing can be
-	// inferred from the wire (plan §5.2).
-	Fabrics []inventory.FabricAttachment `koanf:"fabrics"`
-
-	HeartbeatInterval time.Duration `koanf:"heartbeat_interval"`
-	// ReconnectBase is the first reconnect delay; it backs off from there.
-	ReconnectBase time.Duration `koanf:"reconnect_base"`
-	ReconnectMax  time.Duration `koanf:"reconnect_max"`
-
-	// AdoptReadyTimeout bounds how long an adopted engine is given to answer a
-	// health check before the agent stops waiting on it.
-	//
-	// Shorter than a startup timeout on purpose: this engine is already
-	// running, so it is not loading a model, it is either serving or it is not.
-	AdoptReadyTimeout time.Duration `koanf:"adopt_ready_timeout"`
-
-	// TLS authenticates both ends of the control-plane stream. Leaving it unset
-	// keeps the plaintext behaviour, which only belongs on a trusted network.
-	TLS tlsconf.Config `koanf:"tls"`
-
-	// ModelDirs are the directories this node keeps model weights in, beyond
-	// WeightsCache, which is always looked in. Storage differs at every installation,
-	// so llmgw observes it rather than owning it: a Lustre mount, an image
-	// exported over the fabric, local NVMe and a copy made by hand all look
-	// identical from here, and placement only needs to know whether the file
-	// is present.
-	ModelDirs []string `koanf:"model_dirs"`
-
-	// EnginesDir is where the agent installs engines, one directory per
-	// build. Defaults to /opt/llmgw/engines.
-	EnginesDir string `koanf:"engines_dir"`
-
-	// WeightsCache is where a remote source is downloaded to. Defaults to the
-	// first entry of ModelDirs, because a node that says where its models live
-	// has already answered this question, and otherwise to
-	// /var/lib/llmgw-weights.
-	//
-	// Fetching is the simple arrangement: every node keeps its own copy. The
-	// exotic ones need no code — weights on a shared filer or an image exported
-	// over the fabric are simply present, and present means nothing is fetched.
-	WeightsCache string `koanf:"weights_cache"`
-
-	Discovery discovery.Options `koanf:"discovery"`
-	Engines   engines.Config    `koanf:"engines"`
-	Superv    supervisor.Config `koanf:"supervisor"`
-
-	Version string
-}
-
-// DefaultConfig returns sensible defaults.
-func DefaultConfig() Config {
-	return Config{
-		MgmtInterface:     "eth0",
-		HeartbeatInterval: 5 * time.Second,
-		ReconnectBase:     time.Second,
-		ReconnectMax:      30 * time.Second,
-		AdoptReadyTimeout: 60 * time.Second,
-		Superv:            supervisor.DefaultConfig(),
-		// NVML gives NVLink topology, so prefer it when this build has it.
-		Discovery: discovery.Options{PreferNVML: true},
-	}
-}
-
-func (c *Config) normalize() error {
-	d := DefaultConfig()
-	if err := c.TLS.Validate(); err != nil {
-		return err
-	}
-	if err := c.normalizeServerURL(); err != nil {
-		return err
-	}
-	// Everything below is found rather than written. A file that has to say
-	// which machine it is on, and at what address, is a file that differs on
-	// every node — and a copy taken from one node to the next registers the
-	// second as the first.
-	host, _ := os.Hostname()
-	if c.NodeID == "" {
-		c.NodeID = host
-	}
-	if c.NodeID == "" {
-		return errors.New("node_id is not set and the hostname could not be read")
-	}
-	if c.Hostname == "" {
-		c.Hostname = host
-	}
-	if c.MgmtAddr == "" {
-		c.MgmtAddr = addrToward(c.Server)
-	}
-	if c.MgmtInterface == "" {
-		c.MgmtInterface = interfaceOf(c.MgmtAddr)
-	}
-	if c.MgmtInterface == "" {
-		c.MgmtInterface = d.MgmtInterface
-	}
-	// The directory downloads land in is always one that is looked in. When
-	// they were two settings, a node could fill its disk with fetched weights
-	// that placement never saw — which is what every node here was doing.
-	if c.WeightsCache == "" && len(c.ModelDirs) > 0 {
-		c.WeightsCache = c.ModelDirs[0]
-	}
-	if c.WeightsCache == "" {
-		c.WeightsCache = DefaultWeightsCache
-	}
-	if !slices.Contains(c.ModelDirs, c.WeightsCache) {
-		c.ModelDirs = append(c.ModelDirs, c.WeightsCache)
-	}
-	if c.HeartbeatInterval <= 0 {
-		c.HeartbeatInterval = d.HeartbeatInterval
-	}
-	if c.ReconnectBase <= 0 {
-		c.ReconnectBase = d.ReconnectBase
-	}
-	if c.AdoptReadyTimeout <= 0 {
-		c.AdoptReadyTimeout = d.AdoptReadyTimeout
-	}
-	if c.ReconnectMax < c.ReconnectBase {
-		c.ReconnectMax = max(c.ReconnectBase, d.ReconnectMax)
-	}
-	return nil
-}
-
-// normalizeServerURL makes the configured address a URL connect can use, and
-// refuses the combination that would quietly undo the certificates: a scheme
-// that says plaintext while TLS is configured. Silently connecting anyway would
-// mean a cluster believing it has mutual authentication when it has none.
-func (c *Config) normalizeServerURL() error {
-	// One list, however it was written. Server first when both are given,
-	// because an operator adding a standby to an existing file expects the
-	// one already there to stay preferred.
-	all := make([]string, 0, 1+len(c.Servers))
-	if c.Server != "" {
-		all = append(all, c.Server)
-	}
-	all = append(all, c.Servers...)
-
-	seen := map[string]struct{}{}
-	out := make([]string, 0, len(all))
-	for _, s := range all {
-		fixed, err := normalizeOne(s, c.TLS.Enabled())
-		if err != nil {
-			return err
-		}
-		// A duplicate would be tried twice in a row on failover, which reads
-		// as the rotation not working.
-		if _, dup := seen[fixed]; dup {
-			continue
-		}
-		seen[fixed] = struct{}{}
-		out = append(out, fixed)
-	}
-	if len(out) == 0 {
-		return errors.New("server is required")
-	}
-	c.Servers = out
-	c.Server = out[0]
-	return nil
-}
-
-// DefaultWeightsCache is where fetched weights go when nothing says otherwise.
-const DefaultWeightsCache = "/var/lib/llmgw-weights"
-
-// addrToward is this machine's address on the route to the control plane.
-//
-// The address engines are reached on is whichever one the control plane can
-// reach, and the kernel already knows which that is: a UDP "connection" picks
-// the source address for the route without sending a packet. Asking the
-// interfaces instead would have to guess among docker bridges, VPNs and
-// link-local addresses.
-func addrToward(server string) string {
-	u, err := url.Parse(server)
-	if err != nil || u.Host == "" {
-		return ""
-	}
-	host := u.Hostname()
-	port := u.Port()
-	if port == "" {
-		port = "443"
-	}
-	conn, err := net.Dial("udp", net.JoinHostPort(host, port))
-	if err != nil {
-		return ""
-	}
-	defer conn.Close()
-	if a, ok := conn.LocalAddr().(*net.UDPAddr); ok {
-		return a.IP.String()
-	}
-	return ""
-}
-
-// interfaceOf names the interface that holds an address, or "" when none does.
-func interfaceOf(addr string) string {
-	ip := net.ParseIP(addr)
-	if ip == nil {
-		return ""
-	}
-	ifaces, err := net.Interfaces()
-	if err != nil {
-		return ""
-	}
-	for _, ifc := range ifaces {
-		addrs, err := ifc.Addrs()
-		if err != nil {
-			continue
-		}
-		for _, a := range addrs {
-			if n, ok := a.(*net.IPNet); ok && n.IP.Equal(ip) {
-				return ifc.Name
-			}
-		}
-	}
-	return ""
-}
-
-// normalizeOne applies the scheme rules to a single address.
-func normalizeOne(addr string, secure bool) (string, error) {
-	switch {
-	case strings.HasPrefix(addr, "https://"):
-		if !secure {
-			return "", fmt.Errorf("%s is an https:// address but no tls certificates are configured", addr)
-		}
-	case strings.HasPrefix(addr, "http://"):
-		if secure {
-			return "", fmt.Errorf("tls is configured but %s is an http:// address; use https://", addr)
-		}
-	case strings.Contains(addr, "://"):
-		return "", fmt.Errorf("server %q must be an http:// or https:// address", addr)
-	case secure:
-		return "https://" + addr, nil
-	default:
-		return "http://" + addr, nil
-	}
-	return addr, nil
-}
-
 // Agent runs on a GPU node.
 type Agent struct {
 	cfg  Config
@@ -340,8 +73,11 @@ type Agent struct {
 	lastEngineProbe time.Time
 	probingEngines  bool
 
-	mu  sync.Mutex
-	inv discovery.Inventory
+	mu sync.Mutex
+	// serverList is every control plane to dial: the configured ones, and
+	// any the leader has named since. Read through servers().
+	serverList []string
+	inv        discovery.Inventory
 	// shardReplica remembers which replica a shard belongs to, so a status
 	// report can name both. The control plane keys on the replica.
 	shardReplica map[string]string
@@ -352,6 +88,27 @@ type Agent struct {
 	// failover the old stream's heartbeat loop can still be running when the
 	// new one's starts.
 	lastWeights string
+	// fabricsSent is what the control plane was last told about fabrics.
+	fabricsSent string
+	// sm caches which subnet manager each InfiniBand port sees, asked once
+	// a minute: a management datagram per port per heartbeat is more than
+	// an answer that changes on a failover is worth.
+	sm      map[string]*discovery.SMInfo
+	smErr   map[string]string
+	smAsked time.Time
+	// nvlAsked is when the GPUs' NVLink fabric state was last read.
+	nvlAsked time.Time
+	// sms is every subnet manager on a fabric, asked through one port per
+	// master, keyed by that port; smsErr says why a port could not ask.
+	sms    map[string][]discovery.SMRecord
+	smsErr map[string]string
+	// load measures the machine between heartbeats.
+	load *discovery.LoadSampler
+	// sensors reads heat and power: CPU and board from hwmon and RAPL, GPUs
+	// from nvidia-smi.
+	sensors *discovery.SensorSampler
+	// wear reads what wears a GPU out: remapped rows, PCIe retries, XIDs.
+	wear *discovery.HealthSampler
 
 	// fetchWatchers are the shards waiting on each source, so progress on one
 	// download reaches every shard that asked for it. A fetch is single-flight
@@ -396,6 +153,10 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
 		provStatus:   map[string]provision.Status{},
 		logRing:      ring,
 		provKick:     make(chan struct{}, 1),
+		load:         discovery.NewLoadSampler(),
+		sensors:      discovery.NewSensorSampler(),
+		wear:         discovery.NewHealthSampler(),
+		serverList:   loadControlPlanes(cfg.Servers, cfg.TLS.Enabled()),
 	}
 	a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
 	return a, nil
@@ -423,11 +184,14 @@ func (a *Agent) Run(ctx context.Context) error {
 		a.log.Warn("cannot create the weights directory", "dir", a.cfg.WeightsCache, "err", err)
 	}
 	go a.provisionLoop(ctx)
+	go a.ensureDiags(ctx)
 
 	attempt := 0
 	next := 0
 	for ctx.Err() == nil {
-		server := a.cfg.Servers[next%len(a.cfg.Servers)]
+		// Read each time round: the leader may have named another since.
+		servers := a.servers()
+		server := servers[next%len(servers)]
 		err := a.session(ctx, server)
 		if ctx.Err() != nil {
 			break
@@ -442,12 +206,12 @@ func (a *Agent) Run(ctx context.Context) error {
 		// No wait when there is somewhere else to try: the delay exists to
 		// stop hammering one control plane, and the next address has not been
 		// hammered. With a single address this is the old behaviour exactly.
-		if len(a.cfg.Servers) > 1 && next%len(a.cfg.Servers) != 0 {
+		if len(servers) > 1 && next%len(servers) != 0 {
 			delay = 0
 		}
 		a.log.Warn("disconnected from the control plane, retrying",
 			"err", err, "server", server,
-			"next", a.cfg.Servers[next%len(a.cfg.Servers)],
+			"next", servers[next%len(servers)],
 			"in", delay.Round(time.Millisecond), "attempt", attempt)
 
 		select {
@@ -559,42 +323,21 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) {
 			"shard", o.ShardID, "replica", o.ReplicaID, "err", err)
 		return
 	}
+	// An engine started by an agent older than the GPU check may be one
+	// serving a GPU placement on the CPU. Stopped, so its replacement starts
+	// through the check and fails there in words, if the build still sees
+	// no GPU — rather than going on unnoticed after the upgrade that could
+	// have caught it.
+	if err := a.onItsGPUs(ctx, eng, o.Spec, sh.Status().PID); err != nil {
+		a.log.Error("stopping an adopted engine that computes on the CPU though placed on GPUs",
+			"shard", o.ShardID, "replica", o.ReplicaID, "err", err)
+		_ = a.sup.Stop(ctx, o.ShardID)
+		return
+	}
 	a.log.Info("adopted engine is serving",
 		"shard", o.ShardID, "replica", o.ReplicaID)
 }
 
-// usableFabrics reports the fabrics this node can actually carry traffic on.
-//
-// Loud about what it drops. A fabric silently missing from a registration looks
-// identical to one never configured, and the operator who typed the adapter
-// name is the only person who can fix it — so the reason is logged here, on the
-// machine where both halves are visible, rather than inferred later from a
-// replica that will not converge.
-func (a *Agent) usableFabrics(inv discovery.Inventory) []inventory.FabricAttachment {
-	declared := map[string][]string{}
-	for _, f := range a.cfg.Fabrics {
-		declared[f.Fabric] = f.HCAs
-	}
-
-	var out []inventory.FabricAttachment
-	for _, c := range discovery.CheckFabrics(declared, inv.HCAs) {
-		for _, p := range c.Problems {
-			a.log.Error("a declared fabric adapter cannot carry traffic",
-				"fabric", c.Fabric, "problem", p)
-		}
-		if !c.OK() {
-			a.log.Error("fabric not advertised; no declared adapter on it can carry traffic",
-				"fabric", c.Fabric,
-				"consequence", "no multi-node replica will be placed here")
-			continue
-		}
-		out = append(out, inventory.FabricAttachment{
-			Fabric: c.Fabric, HCAs: c.Usable, RateGbps: c.RateGbps,
-		})
-	}
-	return out
-}
-
 // weights builds the fetcher lazily, so an agent that never starts a shard
 // never needs a cache directory.
 func (a *Agent) weights() *weights.Fetcher {
@@ -682,6 +425,7 @@ func (a *Agent) scan(ctx context.Context) error {
 	opts.ModelDirs = a.cfg.ModelDirs
 	inv, err := discovery.Discover(ctx, opts, a.log)
 	host := discovery.DiscoverHost()
+	host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs)
 	a.mu.Lock()
 	a.inv = inv
 	a.prov.Host = host
@@ -726,6 +470,8 @@ func (a *Agent) session(ctx context.Context, server string) error {
 	// vLLM accepted a vLLM shard and failed it minutes later.
 	reg.EngineVersions = a.engines(parent)
 	reg.EnginesProbed = true
+	reg.InstallableEngines = slices.DeleteFunc(provision.Installable(), a.disabled)
+	reg.Features = []string{FeatureFabricTest}
 	reg.RunningShards = a.runningShards()
 	a.mu.Lock()
 	h := a.prov.Host
@@ -828,6 +574,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
 	// and a node that grew from 7 GB to 32 GB has to be seen to have grown
 	// without anybody restarting anything.
 	hb.MemoryTotalMb = int64(discovery.MemoryTotalMB())
+	hb.CpuCount = int32(runtime.NumCPU())
 
 	a.reprobeEngines()
 	a.mu.Lock()
@@ -850,8 +597,24 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
 	}
 	a.mu.Unlock()
 
-	for _, g := range inv.GPUs {
-		hb.GpuMemoryUsedMb = append(hb.GpuMemoryUsedMb, int32(g.MemoryUsedMB))
+	// Measured now, not carried from registration: a card is shared with
+	// whatever else runs on the machine, and that comes and goes while the
+	// agent runs. The registration figure stays only for a node where
+	// nvidia-smi cannot be asked.
+	if len(inv.GPUs) > 0 {
+		var shards []int
+		for _, st := range a.sup.Statuses() {
+			if st.PID > 0 {
+				shards = append(shards, st.PID)
+			}
+		}
+		hb.GpuForeignMemoryMb, hb.GpuMemoryUsedMb = a.foreignGPUMemory(context.Background(), shards)
+	}
+	if len(hb.GpuMemoryUsedMb) != len(inv.GPUs) {
+		hb.GpuForeignMemoryMb, hb.GpuMemoryUsedMb = nil, nil
+		for _, g := range inv.GPUs {
+			hb.GpuMemoryUsedMb = append(hb.GpuMemoryUsedMb, int32(g.MemoryUsedMB))
+		}
 	}
 
 	// Rescanned here rather than reused from registration. Disk is not a fact
@@ -883,9 +646,74 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
 			})
 		}
 	}
+	hb.Load = a.sampleLoad()
+	a.sampleIB(hb)
+	a.sampleNVLink(hb)
 	return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}}
 }
 
+// sampleLoad measures the machine and each running engine since the last
+// heartbeat. Nil on the first, which has nothing to compare with.
+func (a *Agent) sampleLoad() *llmgwv1.NodeLoad {
+	if a.load == nil {
+		return nil
+	}
+	pids := map[string]int{}
+	for _, st := range a.sup.Statuses() {
+		if st.PID > 0 {
+			pids[st.ID] = st.PID
+		}
+	}
+	l, ok := a.load.Sample(pids)
+	if !ok {
+		return nil
+	}
+	out := &llmgwv1.NodeLoad{
+		CpuPercent: l.CPUPercent, CpuCount: int32(l.CPUCount),
+		Load1: l.Load1, Load5: l.Load5, Load15: l.Load15,
+		MemoryUsedBytes: l.MemUsed, MemoryAvailableBytes: l.MemAvail,
+	}
+	for _, sh := range l.Shards {
+		out.Shards = append(out.Shards, &llmgwv1.ShardLoad{
+			ShardId: sh.ID, CpuCores: sh.CPUCores, MemoryRssBytes: sh.RSSBytes,
+		})
+	}
+	if a.sensors != nil {
+		se := a.sensors.Sample(context.Background())
+		out.CpuTempC, out.CpuPowerW, out.InletTempC = se.CPUTempC, se.CPUPowerW, se.InletTempC
+		var wear map[int]discovery.GPUWear
+		if a.wear != nil && len(se.GPUs) > 0 {
+			wear = map[int]discovery.GPUWear{}
+			for _, w := range a.wear.Sample(context.Background()) {
+				wear[w.Index] = w
+			}
+		}
+		for _, g := range se.GPUs {
+			gs := &llmgwv1.GPUSensor{
+				Index: int32(g.Index), TempC: g.TempC, MemoryTempC: g.MemoryTempC,
+				PowerW: g.PowerW, PowerLimitW: g.PowerLimitW, Throttle: g.Throttle,
+				UtilizationPercent: g.UtilPercent, MemoryUtilizationPercent: g.MemUtilPercent,
+				MemoryUsedMb: g.MemoryUsedMB, MemoryTotalMb: g.MemoryTotalMB,
+				SmClockMhz: g.SMClockMHz, EccUncorrected: g.ECCUncorrected,
+			}
+			if w, ok := wear[g.Index]; ok {
+				gs.HasWear = true
+				gs.PcieReplays, gs.RetiredPagesSbe, gs.RetiredPagesDbe, gs.RetirementPending =
+					w.PCIeReplays, w.RetiredSBE, w.RetiredDBE, w.RetirePending
+				gs.RemappedRowsCorrectable, gs.RemappedRowsUncorrectable, gs.RemapPending, gs.RemapFailure =
+					w.RemappedCorrectable, w.RemappedUncorrectable, w.RemapPending, w.RemapFailure
+				gs.NvlinkCrcErrors, gs.NvlinkReplayErrors, gs.NvlinkRecoveryErrors = w.NVLinkCRC, w.NVLinkReplay, w.NVLinkRecovery
+				gs.XidErrors, gs.LastXid, gs.LastXidText = w.XIDs, int32(w.LastXID), w.LastXIDText
+				if !w.LastXIDAt.IsZero() {
+					gs.LastXidUnix = w.LastXIDAt.Unix()
+				}
+			}
+			out.Gpus = append(out.Gpus, gs)
+		}
+	}
+	return out
+}
+
 // weightsFingerprint is enough to notice a model appearing, disappearing or
 // finishing its download.
 //
@@ -968,6 +796,10 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
 		a.removeWeights(m.RemoveWeights.GetPaths())
 	case *llmgwv1.ControlMessage_Provision:
 		a.setProvision(m.Provision)
+	case *llmgwv1.ControlMessage_Upgrade:
+		go a.upgrade(ctx, m.Upgrade, out)
+	case *llmgwv1.ControlMessage_ControlPlanes:
+		a.setControlPlanes(m.ControlPlanes)
 	case *llmgwv1.ControlMessage_Logs:
 		reply := a.logsReply(m.Logs)
 		// Off the receive loop: a full send queue must not stop commands being
@@ -978,6 +810,33 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
 			case <-ctx.Done():
 			}
 		}()
+	case *llmgwv1.ControlMessage_FabricScan:
+		// A scan takes seconds to minutes: off the receive loop entirely.
+		go func(req *llmgwv1.FabricScanRequest) {
+			start := time.Now()
+			reply := &llmgwv1.FabricScanReply{RequestId: req.GetRequestId()}
+			text, err := scanFabric(ctx, req.GetHca(), int(req.GetPort()))
+			if err != nil {
+				reply.Error = err.Error()
+			} else {
+				reply.Output = text
+			}
+			reply.TookMs = time.Since(start).Milliseconds()
+			select {
+			case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_FabricScan{FabricScan: reply}}:
+			case <-ctx.Done():
+			}
+		}(m.FabricScan)
+	case *llmgwv1.ControlMessage_FabricTest:
+		// Seconds of traffic, and a package install the first time: off the
+		// receive loop too.
+		go func(req *llmgwv1.FabricTestRequest) {
+			reply := a.fabricTest(ctx, req)
+			select {
+			case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_FabricTest{FabricTest: reply}}:
+			case <-ctx.Done():
+			}
+		}(m.FabricTest)
 	case *llmgwv1.ControlMessage_Ack:
 		if !m.Ack.GetOk() {
 			a.log.Warn("control plane rejected a message",
@@ -1047,6 +906,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
 	}
 
 	spec, timeout := agentsvc.ShardSpecFromStart(m)
+	a.localizeSpec(&spec)
 
 	eng, err := a.reg.Get(spec.Profile.Engine)
 	if err != nil {
@@ -1092,6 +952,17 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
 		stop()
 		spec.Profile.ModelPath = path
 	}
+	// Adapters are fetched like the model, and before the engine starts: it
+	// loads them at start, and one missing is a replica that answers for the
+	// base model under the adapter's name.
+	for i, ad := range spec.Profile.Adapters {
+		path, err := a.weights().Ensure(ctx, ad.Path)
+		if err != nil {
+			report(supervisor.Failed, fmt.Sprintf("adapter %s: %v", ad.Name, err), nil)
+			return
+		}
+		spec.Profile.Adapters[i].Path = path
+	}
 
 	cmds, err := eng.Commands(spec)
 	if err != nil {
@@ -1121,6 +992,14 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
 	} else {
 		err = a.sup.WaitAlive(ctx, sh)
 	}
+	if err == nil {
+		err = a.onItsGPUs(ctx, eng, spec, sh.Status().PID)
+		if err != nil {
+			// Serving, but not where it was placed: stopped, so the GPU it
+			// holds in the inventory is not taken by a CPU process.
+			_ = a.sup.Stop(ctx, shardID)
+		}
+	}
 	if err != nil {
 		report(supervisor.Failed, err.Error(), sh.Logs(20))
 		return
@@ -1213,7 +1092,9 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) {
 	if !cfg.Enabled() {
 		return &http.Client{
 			Transport: &http2.Transport{
-				AllowHTTP: true,
+				AllowHTTP:       true,
+				ReadIdleTimeout: streamIdle,
+				PingTimeout:     streamPing,
 				DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
 					return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr)
 				},
@@ -1228,214 +1109,78 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &http.Client{Transport: &http2.Transport{TLSClientConfig: tc}}, nil
+	return &http.Client{Transport: &http2.Transport{
+		TLSClientConfig: tc,
+		ReadIdleTimeout: streamIdle,
+		PingTimeout:     streamPing,
+		// Bounded, as the plaintext branch's is: dialling a control plane
+		// whose machine is off waits out the kernel's connect retries,
+		// about two minutes, before the next address is tried.
+		DialTLSContext: func(ctx context.Context, network, addr string, c *tls.Config) (net.Conn, error) {
+			d := tls.Dialer{NetDialer: &net.Dialer{Timeout: timeout}, Config: c}
+			return d.DialContext(ctx, network, addr)
+		},
+	}}, nil
 }
 
-// engines is what this node can run: what the agent installed and saw run,
-// and what the configuration names and answers. Nothing merely found on the
-// machine — a llama-server on PATH is somebody's, not necessarily one that
-// works, and placing on it is how a replica fails on a node nobody watches.
-func (a *Agent) engines(ctx context.Context) map[string]string {
-	a.engineOnce.Do(func() {
-		vers := a.reg.ProbeConfigured(ctx)
-		a.mu.Lock()
-		a.configuredVers, a.lastEngineProbe = vers, time.Now()
-		a.mu.Unlock()
-		if len(vers) > 0 {
-			a.log.Info("engines named in the configuration",
-				"engines", engines.ProbedNames(vers))
-		}
-	})
-	a.mu.Lock()
-	defer a.mu.Unlock()
-	vers := a.engineReportLocked()
-	// Whatever registration sends is what the control plane now knows, so a
-	// heartbeat repeats it only once it differs.
-	a.enginesSent = enginesFingerprint(vers)
-	return vers
-}
+// A stream to a control plane whose machine stopped dead — powered off, cut
+// off — is never closed: nothing sends the FIN, and the agent's heartbeats sit
+// in the socket's buffer until the kernel gives up retransmitting, fifteen
+// minutes later. Found by switching off a leader: the standby took over in 14
+// seconds and the node stayed with the dead one. An HTTP/2 ping after ten
+// quiet seconds, given five to answer, finds it in a quarter of a minute.
+const (
+	streamIdle = 10 * time.Second
+	streamPing = 5 * time.Second
+)
 
-// engineReportLocked merges the two sources. Installed wins: it is the one
-// that was seen to run. Called with mu held.
-func (a *Agent) engineReportLocked() map[string]string {
-	out := maps.Clone(a.configuredVers)
-	if out == nil {
-		out = map[string]string{}
-	}
-	maps.Copy(out, a.installedVers)
-	return out
-}
-
-// provisionEvery is how often installed engines are checked again. A library
-// removed by an unrelated upgrade is found here rather than by the next
-// replica to start.
-const provisionEvery = 10 * time.Minute
-
-// setProvision takes the control plane's list of wanted engines.
-func (a *Agent) setProvision(p *llmgwv1.Provision) {
-	wants := []provision.Want{}
-	for _, e := range p.GetEngines() {
-		wants = append(wants, provision.Want{Engine: e.GetEngine(), Version: e.GetVersion()})
-	}
-	a.mu.Lock()
-	a.provWants = wants
-	a.mu.Unlock()
-	select {
-	case a.provKick <- struct{}{}:
-	default:
-	}
-}
-
-// provisionLoop installs and re-checks the wanted engines, one pass at a time.
-func (a *Agent) provisionLoop(ctx context.Context) {
-	// After a failure the next pass comes sooner, and backs off: a lock held
-	// by a new machine's own updates clears in a minute or two, while ten
-	// minutes' wait left a node idle for no reason.
-	next := provisionEvery
-	retry := time.Minute
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-a.provKick:
-		case <-time.After(next):
-		}
-		a.mu.Lock()
-		wants := slices.Clone(a.provWants)
-		managed := a.provWants != nil
-		a.mu.Unlock()
-		if !managed {
-			continue
-		}
-
-		statuses := a.prov.Converge(ctx, wants)
-		next = provisionEvery
-		for _, st := range statuses {
-			if st.State == provision.Failed {
-				next, retry = retry, min(retry*2, provisionEvery)
-				break
-			}
-		}
-		if next == provisionEvery {
-			retry = time.Minute
-		}
-		installed := map[string]string{}
-		keep := map[string]bool{}
-		for _, st := range statuses {
-			keep[st.Engine] = true
-			if st.State == provision.Ready {
-				a.reg.UseInstalled(st.Engine, st.Binary)
-				installed[st.Engine] = st.Version
-			}
-		}
-		a.mu.Lock()
-		if enginesFingerprint(installed) != enginesFingerprint(a.installedVers) {
-			a.log.Info("engines installed and running", "engines", engines.ProbedNames(installed))
-		}
-		a.installedVers = installed
-		// An engine no longer asked for is no longer reported on.
-		for e := range a.provStatus {
-			if !keep[e] {
-				delete(a.provStatus, e)
-			}
-		}
-		a.mu.Unlock()
-	}
-}
-
-// provisionStatus records a change in one engine's installation.
-func (a *Agent) provisionStatus(st provision.Status) {
-	a.mu.Lock()
-	prev, had := a.provStatus[st.Engine]
-	a.provStatus[st.Engine] = st
-	a.mu.Unlock()
-	if !had || prev.State != st.State || prev.Detail != st.Detail {
-		lvl := slog.LevelInfo
-		if st.State == provision.Failed || st.State == provision.Unsupported {
-			lvl = slog.LevelWarn
-		}
-		a.log.Log(context.Background(), lvl, "engine installation",
-			"engine", st.Engine, "version", st.Version, "variant", st.Variant,
-			"state", st.State, "detail", st.Detail)
-	}
-}
-
-func provisionFingerprint(m map[string]provision.Status) string {
-	var b strings.Builder
-	for _, k := range slices.Sorted(maps.Keys(m)) {
-		st := m[k]
-		fmt.Fprintf(&b, "%s|%s|%s|%s|%s\n", st.Engine, st.Version, st.Variant, st.State, st.Detail)
-	}
-	return b.String()
-}
-
-// engineReprobe is how often the installed engines are looked for again.
+// localizeSpec fills in what only this node knows.
 //
-// Minutes, not seconds: an engine is installed by somebody at a keyboard, so a
-// minute's delay in noticing is invisible, while SGLang's probe starts a
-// Python interpreter that imports it.
-const engineReprobe = time.Minute
-
-// reprobeEngines looks for installed engines again when it is time, off the
-// heartbeat goroutine so a slow probe never delays a heartbeat.
-func (a *Agent) reprobeEngines() {
-	a.mu.Lock()
-	due := !a.probingEngines && !a.lastEngineProbe.IsZero() &&
-		time.Since(a.lastEngineProbe) >= engineReprobe
-	if due {
-		a.probingEngines = true
+// The interface NCCL bootstraps over is this node's, not the cluster's: the
+// control plane sends one name for every node, and a node whose management
+// network is on ens3 rather than eth0 would have NCCL bind nothing. And on
+// RoCE, which GID entry routes is a property of these adapters.
+func (a *Agent) localizeSpec(spec *engines.ShardSpec) {
+	if a.cfg.MgmtInterface != "" {
+		spec.MgmtInterface = a.cfg.MgmtInterface
 	}
-	a.mu.Unlock()
-	if !due {
+	if len(spec.HCAs) == 0 {
 		return
 	}
-	go func() {
-		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-		defer cancel()
-		vers := a.reg.ProbeConfigured(ctx)
-
-		a.mu.Lock()
-		changed := enginesFingerprint(vers) != enginesFingerprint(a.configuredVers)
-		a.configuredVers, a.lastEngineProbe, a.probingEngines = vers, time.Now(), false
-		a.mu.Unlock()
-		if changed {
-			a.log.Info("engines named in the configuration changed",
-				"engines", engines.ProbedNames(vers))
+	a.mu.Lock()
+	hcas := a.inv.HCAs
+	a.mu.Unlock()
+	want := map[string]bool{}
+	for _, h := range spec.HCAs {
+		want[h] = true
+	}
+	for _, h := range hcas {
+		if !want[h.Name] {
+			continue
 		}
-	}()
-}
-
-func enginesFingerprint(vers map[string]string) string {
-	var b strings.Builder
-	for _, k := range slices.Sorted(maps.Keys(vers)) {
-		fmt.Fprintf(&b, "%s=%s\n", k, vers[k])
-	}
-	return b.String()
-}
-
-// logsReply answers a request for one of this node's logs.
-func (a *Agent) logsReply(req *llmgwv1.LogsRequest) *llmgwv1.LogsReply {
-	n := int(req.GetLines())
-	if n <= 0 {
-		n = 200
-	}
-	n = min(n, agentLogLines)
-	shards := a.sup.IDs()
-	slices.Sort(shards)
-	out := &llmgwv1.LogsReply{
-		RequestId: req.GetRequestId(),
-		Sources:   append([]string{"agent"}, shards...),
-	}
-	switch src := req.GetSource(); src {
-	case "", "agent":
-		out.Lines = a.logRing.tail(n)
-	default:
-		sh, err := a.sup.Get(src)
-		if err != nil {
-			out.Error = "no engine " + src + " runs on this node"
-			return out
+		for _, p := range h.Ports {
+			if p.Active() && p.RoCEv2GID >= 0 {
+				idx := p.RoCEv2GID
+				spec.GIDIndex = &idx
+				return
+			}
 		}
-		out.Lines = sh.Logs(n)
 	}
-	return out
+}
+
+// ResolveManagement is the address and interface this node's agent would use,
+// found the way normalize finds them, for a tool that reports them.
+func ResolveManagement(c Config) (addr, iface string) {
+	addr, iface = c.MgmtAddr, c.MgmtInterface
+	server := c.Server
+	if server == "" && len(c.Servers) > 0 {
+		server = c.Servers[0]
+	}
+	if addr == "" {
+		addr = addrToward(server)
+	}
+	if iface == "" {
+		iface = interfaceOf(addr)
+	}
+	return addr, iface
 }

internal/agent/agent_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/tls"
 	"errors"
+	"fmt"
 	"log/slog"
 	"net"
 	"net/http"
@@ -22,6 +23,8 @@ import (
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
 
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
@@ -397,11 +400,16 @@ func TestHeartbeatKeepsNodeAlive(t *testing.T) {
 	n, _ := h.inv.Get("n1")
 	first := n.LastSeen
 
-	time.Sleep(200 * time.Millisecond)
-	n, _ = h.inv.Get("n1")
-	if !n.LastSeen.After(first) {
-		t.Error("heartbeats are not refreshing LastSeen; the node would be declared lost")
+	// Waited for rather than slept through: four heartbeat intervals was not
+	// always enough on a loaded runner for one to have landed.
+	deadline := time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		if n, _ = h.inv.Get("n1"); n.LastSeen.After(first) {
+			return
+		}
+		time.Sleep(20 * time.Millisecond)
 	}
+	t.Error("heartbeats are not refreshing LastSeen; the node would be declared lost")
 }
 
 func TestStartAndStopShardEndToEnd(t *testing.T) {
@@ -756,3 +764,157 @@ func TestAFabricRateIsTheSlowestActiveLink(t *testing.T) {
 		}
 	}
 }
+
+func TestAPortGoingDownIsToldToTheControlPlaneOnce(t *testing.T) {
+	// Checked only at registration, a port going down went unnoticed until
+	// the agent restarted, and the node went on being offered multi-node
+	// work that would hang on its first collective.
+	root := ibTreeWithActivePorts(t, "mlx5_0")
+	h := start(t, func(c *Config) {
+		c.Discovery.IBRoot = root
+		c.Fabrics = []inventory.FabricAttachment{{Fabric: "ib-a", HCAs: []string{"mlx5_0"}}}
+	})
+	if n, _ := h.inv.Get("n1"); !n.HasFabric("ib-a") {
+		t.Fatalf("the fabric should be advertised at registration: %+v", n.Fabrics)
+	}
+
+	hb := h.ag.heartbeat().GetHeartbeat()
+	if hb.GetFabricsChanged() || len(hb.GetIbPorts()) != 1 || hb.GetIbPorts()[0].GetState() != "ACTIVE" {
+		t.Fatalf("a quiet heartbeat should carry the port and no change: %+v", hb)
+	}
+
+	if err := os.WriteFile(filepath.Join(root, "mlx5_0", "ports", "1", "state"), []byte("1: DOWN\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	hb = h.ag.heartbeat().GetHeartbeat()
+	if !hb.GetFabricsChanged() || len(hb.GetFabrics()) != 0 {
+		t.Errorf("the port going down should withdraw the fabric: changed=%v fabrics=%v",
+			hb.GetFabricsChanged(), hb.GetFabrics())
+	}
+	if hb = h.ag.heartbeat().GetHeartbeat(); hb.GetFabricsChanged() {
+		t.Error("the change should be sent once, not on every heartbeat")
+	}
+}
+
+func TestAShardUsesThisNodesInterfaceAndRoCEIndex(t *testing.T) {
+	// The control plane sends one interface name for every node; a node whose
+	// management network is on another had NCCL bind nothing. And which RoCE
+	// GID entry routes is a property of this node's adapters.
+	a := &Agent{cfg: Config{MgmtInterface: "ens3"}}
+	a.inv.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{
+		{Num: 1, State: "ACTIVE", LinkLayer: discovery.LinkEthernet, RoCEv2GID: 3}}}}
+	spec := engines.ShardSpec{MgmtInterface: "eth0", HCAs: []string{"mlx5_0"}}
+	a.localizeSpec(&spec)
+	if spec.MgmtInterface != "ens3" {
+		t.Errorf("interface = %q", spec.MgmtInterface)
+	}
+	if spec.GIDIndex == nil || *spec.GIDIndex != 3 {
+		t.Errorf("GID index = %v", spec.GIDIndex)
+	}
+
+	ib := engines.ShardSpec{HCAs: []string{"mlx5_1"}}
+	a.inv.HCAs = append(a.inv.HCAs, discovery.HCA{Name: "mlx5_1", Ports: []discovery.Port{
+		{Num: 1, State: "ACTIVE", LinkLayer: discovery.LinkInfiniBand, RoCEv2GID: -1}}})
+	a.localizeSpec(&ib)
+	if ib.GIDIndex != nil {
+		t.Error("an InfiniBand shard was given a GID index")
+	}
+}
+
+func TestTheManagementInterfaceIsFoundNotAssumed(t *testing.T) {
+	// It was defaulted to eth0 before it could be looked for, so it never
+	// was: a node on ens3 or bond0 had NCCL bootstrap over nothing.
+	cfg := DefaultConfig()
+	cfg.Server = "http://127.0.0.1:7443"
+	cfg.NodeID = "n1"
+	cfg.MgmtAddr = "127.0.0.1"
+	if err := cfg.normalize(); err != nil {
+		t.Fatal(err)
+	}
+	if cfg.MgmtInterface == "" || cfg.MgmtInterface == "eth0" {
+		t.Errorf("interface = %q, want the loopback interface holding 127.0.0.1", cfg.MgmtInterface)
+	}
+}
+
+func TestEachFabricsSubnetManagersAreAskedOnce(t *testing.T) {
+	// Two adapters on one fabric see one master. Its managers are listed
+	// through one of them — the same list twice is a query too many — and
+	// every port says which master it sees.
+	oldSM, oldSMs := querySM, querySMs
+	t.Cleanup(func() { querySM, querySMs = oldSM, oldSMs })
+	querySM = func(string, int) (*discovery.SMInfo, error) {
+		return &discovery.SMInfo{GUID: "0x01", LID: 1, State: "SMINFO_MASTER"}, nil
+	}
+	lists := 0
+	querySMs = func(string, int) ([]discovery.SMRecord, error) {
+		lists++
+		return []discovery.SMRecord{{GUID: "0x01", State: "master"}, {GUID: "0x02", State: "standby"}}, nil
+	}
+	h := start(t, func(c *Config) {
+		c.Discovery.IBRoot = ibTreeWithActivePorts(t, "mlx5_0", "mlx5_1")
+		c.Fabrics = []inventory.FabricAttachment{{Fabric: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}}}
+	})
+	hb := h.ag.heartbeat().GetHeartbeat()
+	withList := 0
+	for _, p := range hb.GetIbPorts() {
+		if p.GetSmGuid() != "0x01" {
+			t.Errorf("%s sees master %q", p.GetHca(), p.GetSmGuid())
+		}
+		if len(p.GetSubnetManagers()) > 0 {
+			withList++
+		}
+	}
+	if lists != 1 || withList != 1 {
+		t.Errorf("listed %d times, carried on %d ports; want once, on one", lists, withList)
+	}
+}
+
+func TestAFabricScanIsAskedForAndAnswered(t *testing.T) {
+	// One node per fabric maps it; the control plane asks and parses.
+	old := scanFabric
+	t.Cleanup(func() { scanFabric = old })
+	var asked string
+	scanFabric = func(_ context.Context, hca string, port int) (string, error) {
+		asked = fmt.Sprintf("%s/%d", hca, port)
+		return "Switch\t36 \"S-00000000000000a1\"\t\t# \"MF0;leaf-a:QM8700/U1\" enhanced port 0 lid 1 lmc 0\n", nil
+	}
+	h := start(t)
+	r, err := h.svc.ScanFabric(t.Context(), "n1", "mlx5_0", 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if asked != "mlx5_0/1" || !strings.Contains(r.GetOutput(), "leaf-a") || r.GetError() != "" {
+		t.Errorf("asked %q, got %+v", asked, r)
+	}
+}
+
+func TestABandwidthTestIsRunOnTheAdapterAskedFor(t *testing.T) {
+	oldRun, oldHave := runWriteBW, haveWriteBW
+	t.Cleanup(func() { runWriteBW, haveWriteBW = oldRun, oldHave })
+	haveWriteBW = func() bool { return true }
+	var got discovery.WriteBW
+	runWriteBW = func(_ context.Context, w discovery.WriteBW) (float64, string, error) {
+		got = w
+		return 388.2, " #bytes  #iterations  BW peak[Gb/sec]  BW average[Gb/sec]\n", nil
+	}
+	h := start(t, func(c *Config) { c.Discovery.IBRoot = ibTreeWithActivePorts(t, "mlx5_0") })
+
+	r, err := h.svc.FabricTest(t.Context(), "n1", &llmgwv1.FabricTestRequest{
+		Role: "client", Hca: "mlx5_0", Port: 1, Peer: "10.0.0.2", TcpPort: 18515, Seconds: 3})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if r.GetGbps() != 388.2 || r.GetError() != "" {
+		t.Errorf("reply %+v", r)
+	}
+	if got.HCA != "mlx5_0" || got.Peer != "10.0.0.2" || got.TCPPort != 18515 || got.GIDIndex != -1 {
+		t.Errorf("ran %+v", got)
+	}
+
+	// An adapter the node does not have is said so, without running anything.
+	got = discovery.WriteBW{}
+	r, _ = h.svc.FabricTest(t.Context(), "n1", &llmgwv1.FabricTestRequest{Role: "server", Hca: "mlx5_9", Port: 1, Seconds: 1})
+	if !strings.Contains(r.GetError(), "no port mlx5_9/1") || got.HCA != "" {
+		t.Errorf("reply %+v, ran %+v", r, got)
+	}
+}

internal/agent/config.go

@@ -0,0 +1,286 @@
+package agent
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"slices"
+	"strings"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
+)
+
+// Config configures the agent.
+type Config struct {
+	// NodeID defaults to the hostname. It must equal the common name of the
+	// agent's certificate, so set it only where the two differ.
+	NodeID string `koanf:"node_id"`
+	// Server is the control plane, and Servers is the same thing when there is
+	// more than one. Both are accepted: every configuration written before
+	// there could be a second control plane names one address, and none of
+	// them should have to change.
+	//
+	// The order is tried in order, and a refusal moves to the next. Only the
+	// leader accepts an agent, so a standby is skipped in the time it takes to
+	// be told so — which is what makes failover a reconnect rather than an
+	// edit to every node's configuration.
+	Server  string   `koanf:"server"`
+	Servers []string `koanf:"servers"`
+	// Hostname defaults to the machine's.
+	Hostname string `koanf:"hostname"`
+	// MgmtAddr is the address engines are reached on. It defaults to this
+	// machine's address on the route to the control plane, which is the one
+	// the control plane and the router can reach.
+	MgmtAddr string `koanf:"mgmt_addr"`
+	// Labels are free-form facts about a node for a profile's node_selector.
+	// Nothing needs them: memory, GPUs, engines and disk are measured, and a
+	// label restating one of those goes stale the day the machine changes.
+	Labels map[string]string `koanf:"labels"`
+
+	// MgmtInterface is the ethernet interface NCCL bootstraps over. It
+	// defaults to the interface holding MgmtAddr, then eth0.
+	MgmtInterface string `koanf:"mgmt_interface"`
+	// Fabrics labels this node's HCAs. It is explicit on purpose: OpenSM's
+	// default subnet prefix is the same on every fabric, so nothing can be
+	// inferred from the wire (plan §5.2).
+	Fabrics []inventory.FabricAttachment `koanf:"fabrics"`
+
+	HeartbeatInterval time.Duration `koanf:"heartbeat_interval"`
+	// ReconnectBase is the first reconnect delay; it backs off from there.
+	ReconnectBase time.Duration `koanf:"reconnect_base"`
+	ReconnectMax  time.Duration `koanf:"reconnect_max"`
+
+	// AdoptReadyTimeout bounds how long an adopted engine is given to answer a
+	// health check before the agent stops waiting on it.
+	//
+	// Shorter than a startup timeout on purpose: this engine is already
+	// running, so it is not loading a model, it is either serving or it is not.
+	AdoptReadyTimeout time.Duration `koanf:"adopt_ready_timeout"`
+
+	// TLS authenticates both ends of the control-plane stream. Leaving it unset
+	// keeps the plaintext behaviour, which only belongs on a trusted network.
+	TLS tlsconf.Config `koanf:"tls"`
+
+	// ModelDirs are the directories this node keeps model weights in, beyond
+	// WeightsCache, which is always looked in. Storage differs at every installation,
+	// so llmgw observes it rather than owning it: a Lustre mount, an image
+	// exported over the fabric, local NVMe and a copy made by hand all look
+	// identical from here, and placement only needs to know whether the file
+	// is present.
+	ModelDirs []string `koanf:"model_dirs"`
+
+	// EnginesDir is where the agent installs engines, one directory per
+	// build. Defaults to /opt/llmgw/engines.
+	EnginesDir string `koanf:"engines_dir"`
+
+	// WeightsCache is where a remote source is downloaded to. Defaults to the
+	// first entry of ModelDirs, because a node that says where its models live
+	// has already answered this question, and otherwise to
+	// /var/lib/llmgw-weights.
+	//
+	// Fetching is the simple arrangement: every node keeps its own copy. The
+	// exotic ones need no code — weights on a shared filer or an image exported
+	// over the fabric are simply present, and present means nothing is fetched.
+	WeightsCache string `koanf:"weights_cache"`
+
+	Discovery discovery.Options `koanf:"discovery"`
+	Engines   engines.Config    `koanf:"engines"`
+	Superv    supervisor.Config `koanf:"supervisor"`
+
+	Version string
+}
+
+// DefaultConfig returns sensible defaults.
+func DefaultConfig() Config {
+	// No MgmtInterface: normalize finds the one holding the management
+	// address, and falls back to eth0 only when it cannot. Defaulted here, it
+	// was never found — every node said eth0 unless told otherwise, and a node
+	// on ens3 or bond0 had NCCL bootstrap over an interface it does not have.
+	return Config{
+		HeartbeatInterval: 5 * time.Second,
+		ReconnectBase:     time.Second,
+		ReconnectMax:      30 * time.Second,
+		AdoptReadyTimeout: 60 * time.Second,
+		Superv:            supervisor.DefaultConfig(),
+		// NVML gives NVLink topology, so prefer it when this build has it.
+		Discovery: discovery.Options{PreferNVML: true},
+	}
+}
+
+func (c *Config) normalize() error {
+	d := DefaultConfig()
+	if err := c.TLS.Validate(); err != nil {
+		return err
+	}
+	if err := c.normalizeServerURL(); err != nil {
+		return err
+	}
+	// Everything below is found rather than written. A file that has to say
+	// which machine it is on, and at what address, is a file that differs on
+	// every node — and a copy taken from one node to the next registers the
+	// second as the first.
+	host, _ := os.Hostname()
+	if c.NodeID == "" {
+		c.NodeID = host
+	}
+	if c.NodeID == "" {
+		return errors.New("node_id is not set and the hostname could not be read")
+	}
+	if c.Hostname == "" {
+		c.Hostname = host
+	}
+	if c.MgmtAddr == "" {
+		c.MgmtAddr = addrToward(c.Server)
+	}
+	if c.MgmtInterface == "" {
+		c.MgmtInterface = interfaceOf(c.MgmtAddr)
+	}
+	if c.MgmtInterface == "" {
+		c.MgmtInterface = "eth0"
+	}
+	// The directory downloads land in is always one that is looked in. When
+	// they were two settings, a node could fill its disk with fetched weights
+	// that placement never saw — which is what every node here was doing.
+	if c.WeightsCache == "" && len(c.ModelDirs) > 0 {
+		c.WeightsCache = c.ModelDirs[0]
+	}
+	if c.WeightsCache == "" {
+		c.WeightsCache = DefaultWeightsCache
+	}
+	if !slices.Contains(c.ModelDirs, c.WeightsCache) {
+		c.ModelDirs = append(c.ModelDirs, c.WeightsCache)
+	}
+	if c.HeartbeatInterval <= 0 {
+		c.HeartbeatInterval = d.HeartbeatInterval
+	}
+	if c.ReconnectBase <= 0 {
+		c.ReconnectBase = d.ReconnectBase
+	}
+	if c.AdoptReadyTimeout <= 0 {
+		c.AdoptReadyTimeout = d.AdoptReadyTimeout
+	}
+	if c.ReconnectMax < c.ReconnectBase {
+		c.ReconnectMax = max(c.ReconnectBase, d.ReconnectMax)
+	}
+	return nil
+}
+
+// normalizeServerURL makes the configured address a URL connect can use, and
+// refuses the combination that would quietly undo the certificates: a scheme
+// that says plaintext while TLS is configured. Silently connecting anyway would
+// mean a cluster believing it has mutual authentication when it has none.
+func (c *Config) normalizeServerURL() error {
+	// One list, however it was written. Server first when both are given,
+	// because an operator adding a standby to an existing file expects the
+	// one already there to stay preferred.
+	all := make([]string, 0, 1+len(c.Servers))
+	if c.Server != "" {
+		all = append(all, c.Server)
+	}
+	all = append(all, c.Servers...)
+
+	seen := map[string]struct{}{}
+	out := make([]string, 0, len(all))
+	for _, s := range all {
+		fixed, err := normalizeOne(s, c.TLS.Enabled())
+		if err != nil {
+			return err
+		}
+		// A duplicate would be tried twice in a row on failover, which reads
+		// as the rotation not working.
+		if _, dup := seen[fixed]; dup {
+			continue
+		}
+		seen[fixed] = struct{}{}
+		out = append(out, fixed)
+	}
+	if len(out) == 0 {
+		return errors.New("server is required")
+	}
+	c.Servers = out
+	c.Server = out[0]
+	return nil
+}
+
+// DefaultWeightsCache is where fetched weights go when nothing says otherwise.
+const DefaultWeightsCache = "/var/lib/llmgw-weights"
+
+// addrToward is this machine's address on the route to the control plane.
+//
+// The address engines are reached on is whichever one the control plane can
+// reach, and the kernel already knows which that is: a UDP "connection" picks
+// the source address for the route without sending a packet. Asking the
+// interfaces instead would have to guess among docker bridges, VPNs and
+// link-local addresses.
+func addrToward(server string) string {
+	u, err := url.Parse(server)
+	if err != nil || u.Host == "" {
+		return ""
+	}
+	host := u.Hostname()
+	port := u.Port()
+	if port == "" {
+		port = "443"
+	}
+	conn, err := net.Dial("udp", net.JoinHostPort(host, port))
+	if err != nil {
+		return ""
+	}
+	defer conn.Close()
+	if a, ok := conn.LocalAddr().(*net.UDPAddr); ok {
+		return a.IP.String()
+	}
+	return ""
+}
+
+// interfaceOf names the interface that holds an address, or "" when none does.
+func interfaceOf(addr string) string {
+	ip := net.ParseIP(addr)
+	if ip == nil {
+		return ""
+	}
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return ""
+	}
+	for _, ifc := range ifaces {
+		addrs, err := ifc.Addrs()
+		if err != nil {
+			continue
+		}
+		for _, a := range addrs {
+			if n, ok := a.(*net.IPNet); ok && n.IP.Equal(ip) {
+				return ifc.Name
+			}
+		}
+	}
+	return ""
+}
+
+// normalizeOne applies the scheme rules to a single address.
+func normalizeOne(addr string, secure bool) (string, error) {
+	switch {
+	case strings.HasPrefix(addr, "https://"):
+		if !secure {
+			return "", fmt.Errorf("%s is an https:// address but no tls certificates are configured", addr)
+		}
+	case strings.HasPrefix(addr, "http://"):
+		if secure {
+			return "", fmt.Errorf("tls is configured but %s is an http:// address; use https://", addr)
+		}
+	case strings.Contains(addr, "://"):
+		return "", fmt.Errorf("server %q must be an http:// or https:// address", addr)
+	case secure:
+		return "https://" + addr, nil
+	default:
+		return "http://" + addr, nil
+	}
+	return addr, nil
+}

internal/agent/controlplanes.go

@@ -0,0 +1,77 @@
+package agent
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"slices"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+)
+
+// controlPlanesFile keeps the control planes the leader last named, beside
+// the agent's configuration rather than in it: the configuration is the
+// operator's, and a list rewritten in it would be a file nobody wrote.
+var controlPlanesFile = "/var/lib/llmgw-agent/control-planes.json"
+
+// servers is every address to dial, in the order to try them: the
+// configuration's, then any the leader named that it lacks.
+func (a *Agent) servers() []string {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	if len(a.serverList) == 0 {
+		return a.cfg.Servers
+	}
+	return a.serverList
+}
+
+// mergeServers puts the leader's list after the configured one, without
+// repeats, and says whether that changed anything.
+func mergeServers(configured, told []string, secure bool) []string {
+	out := slices.Clone(configured)
+	for _, s := range told {
+		fixed, err := normalizeOne(s, secure)
+		if err != nil {
+			continue
+		}
+		if !slices.Contains(out, fixed) {
+			out = append(out, fixed)
+		}
+	}
+	return out
+}
+
+// setControlPlanes takes the leader's list: used from the next reconnect,
+// and kept for the next start.
+func (a *Agent) setControlPlanes(m *llmgwv1.ControlPlanes) {
+	told := m.GetServers()
+	merged := mergeServers(a.cfg.Servers, told, a.cfg.TLS.Enabled())
+	a.mu.Lock()
+	changed := !slices.Equal(merged, a.serverList)
+	a.serverList = merged
+	a.mu.Unlock()
+	if !changed {
+		return
+	}
+	a.log.Info("the control planes to dial", "servers", merged)
+	b, _ := json.Marshal(told)
+	if err := os.MkdirAll(filepath.Dir(controlPlanesFile), 0o755); err == nil {
+		tmp := controlPlanesFile + ".new"
+		if err := os.WriteFile(tmp, b, 0o644); err == nil {
+			_ = os.Rename(tmp, controlPlanesFile)
+		}
+	}
+}
+
+// loadControlPlanes is the list kept from the last time the leader named it.
+func loadControlPlanes(configured []string, secure bool) []string {
+	b, err := os.ReadFile(controlPlanesFile)
+	if err != nil {
+		return configured
+	}
+	var told []string
+	if json.Unmarshal(b, &told) != nil {
+		return configured
+	}
+	return mergeServers(configured, told, secure)
+}

internal/agent/controlplanes_test.go

@@ -0,0 +1,41 @@
+package agent
+
+import (
+	"log/slog"
+	"path/filepath"
+	"slices"
+	"testing"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
+)
+
+func TestAControlPlaneTheLeaderNamesIsDialledAndRemembered(t *testing.T) {
+	old := controlPlanesFile
+	controlPlanesFile = filepath.Join(t.TempDir(), "control-planes.json")
+	t.Cleanup(func() { controlPlanesFile = old })
+
+	configured := []string{"https://10.0.0.140:7443"}
+	// TLS on, as every onboarded node has it: an agent without it refuses
+	// https:// addresses, and the leader's are https://.
+	a := &Agent{cfg: Config{Servers: configured, TLS: tlsconf.Config{CA: "/etc/llmgw/ca.pem"}}, log: slog.New(slog.DiscardHandler)}
+	if got := a.servers(); !slices.Equal(got, configured) {
+		t.Fatalf("before anything is named: %v", got)
+	}
+	// The configured one stays first; the new one is added after it, once.
+	a.setControlPlanes(&llmgwv1.ControlPlanes{Servers: []string{"https://10.0.0.140:7443", "https://10.0.0.150:7443"}})
+	want := []string{"https://10.0.0.140:7443", "https://10.0.0.150:7443"}
+	if got := a.servers(); !slices.Equal(got, want) {
+		t.Errorf("after the leader named two: %v", got)
+	}
+	// And after a restart, before the leader says anything.
+	if got := loadControlPlanes(configured, true); !slices.Equal(got, want) {
+		t.Errorf("after a restart: %v", got)
+	}
+	// A list that drops the configured one does not drop it: the
+	// configuration is the operator's.
+	a.setControlPlanes(&llmgwv1.ControlPlanes{Servers: []string{"https://10.0.0.150:7443"}})
+	if got := a.servers(); got[0] != "https://10.0.0.140:7443" {
+		t.Errorf("the configured server was dropped: %v", got)
+	}
+}

internal/agent/disabled_test.go

@@ -0,0 +1,32 @@
+package agent
+
+import (
+	"testing"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+)
+
+func TestAnEngineDisabledOnTheNodeIsNeitherInstalledNorAdvertised(t *testing.T) {
+	a := &Agent{}
+	a.cfg.Engines.Disabled = []string{"vllm"}
+	a.setProvision(&llmgwv1.Provision{Engines: []*llmgwv1.EngineWant{
+		{Engine: "llamacpp", Version: "b1"}, {Engine: "vllm", Version: "0.30.0"},
+	}})
+	if len(a.provWants) != 1 || a.provWants[0].Engine != "llamacpp" {
+		t.Errorf("wants %+v: vLLM is disabled here and must not be installed", a.provWants)
+	}
+	a.configuredVers = map[string]string{"vllm": "0.25.0"}
+	a.installedVers = map[string]string{"llamacpp": "b1"}
+	if got := a.engineReportLocked(); got["vllm"] != "" || got["llamacpp"] != "b1" {
+		t.Errorf("advertised %v: a disabled engine found on disk is still not offered", got)
+	}
+}
+
+func TestAnEngineNamedByHandIsNotInstalledOverIt(t *testing.T) {
+	a := &Agent{reg: engines.NewRegistry(engines.Config{LlamaCpp: engines.LlamaCppConfig{Bin: "/opt/own/llama-server"}})}
+	a.setProvision(&llmgwv1.Provision{Engines: []*llmgwv1.EngineWant{{Engine: "llamacpp", Version: "b1"}}})
+	if len(a.provWants) != 0 {
+		t.Errorf("wants %+v: an installed build would be used over the operator's own", a.provWants)
+	}
+}

internal/agent/discovery/discover.go

@@ -21,8 +21,9 @@ type Options struct {
 	NvidiaSMI string `koanf:"nvidia_smi"`
 	// IBRoot is the sysfs InfiniBand directory.
 	IBRoot string `koanf:"ib_root"`
-	// PreferNVML uses the NVML backend when this binary has one. It is the only
-	// source of NVLink topology.
+	// PreferNVML uses the NVML backend when this binary has one. NVLink
+	// topology is read from nvidia-smi topo -m on every build; NVML, where it
+	// is used, is the more exact source of the peers.
 	PreferNVML bool `koanf:"prefer_nvml"`
 	// SkipGPUs omits GPU discovery. A node without nvidia-smi is already
 	// found to have none, so this is for one whose driver is broken.
@@ -64,8 +65,7 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
 				inv.GPUs, inv.Source = gpus, "nvml"
 				break
 			}
-			log.Warn("NVML discovery failed, falling back to nvidia-smi; "+
-				"NVLink topology will be unavailable", "err", err)
+			log.Warn("NVML discovery failed, falling back to nvidia-smi", "err", err)
 			fallthrough
 		default:
 			gpus, err := DiscoverGPUsSMI(ctx, opts.NvidiaSMI)
@@ -74,6 +74,26 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
 			}
 			inv.GPUs, inv.Source = gpus, "nvidia-smi"
 		}
+		// NVLink and which adapter sits nearest each card, for either path:
+		// NVML knows NVLink but not the adapters, nvidia-smi's query knows
+		// neither. A failure costs the topology and nothing else — the cards
+		// are still cards.
+		if len(inv.GPUs) > 0 {
+			topo, err := DiscoverTopology(ctx, opts.NvidiaSMI)
+			if err != nil {
+				log.Warn("could not read the GPU topology; tensor-parallel groups "+
+					"will be chosen without NVLink", "err", err)
+			} else {
+				topo.Apply(inv.GPUs)
+			}
+			// Multi-node NVLink: which domain each card has joined.
+			if fab, err := DiscoverGPUFabric(ctx, opts.NvidiaSMI); err != nil {
+				log.Warn("could not read the GPUs' NVLink fabric state", "err", err)
+			} else {
+				ApplyFabric(inv.GPUs, fab)
+			}
+			inv.IMEX = DiscoverIMEX(ctx)
+		}
 	}
 
 	hcas, err := DiscoverIB(opts.IBRoot)

internal/agent/discovery/fabrics.go

@@ -10,6 +10,8 @@ import (
 // has.
 type FabricCheck struct {
 	Fabric string
+	// Declared are the adapters the configuration names for it.
+	Declared []string
 	// Usable are the declared adapters that exist and have an active port.
 	Usable []string
 	// RateGbps is the slowest active port among them, because a collective
@@ -49,7 +51,7 @@ func CheckFabrics(declared map[string][]string, discovered []HCA) []FabricCheck
 
 	out := make([]FabricCheck, 0, len(names))
 	for _, fabric := range names {
-		c := FabricCheck{Fabric: fabric}
+		c := FabricCheck{Fabric: fabric, Declared: declared[fabric]}
 		for _, want := range declared[fabric] {
 			hca, found := byName[want]
 			if !found {

internal/agent/discovery/gpu.go

@@ -38,6 +38,13 @@ type GPU struct {
 	// NVLinkPeers are the indices of GPUs reachable over NVLink. Within a node
 	// a TP group should stay inside one NVLink domain.
 	NVLinkPeers []int `json:"nvlink_peers,omitempty"`
+	// NICPaths is how far each network adapter is from this GPU, by HCA name:
+	// PIX, PXB, PHB, NODE or SYS, as nvidia-smi topo reports it. What decides
+	// which adapter a GPU's share of a multi-node job should go through.
+	NICPaths map[string]string `json:"nic_paths,omitempty"`
+	// Fabric is the GPU's multi-node NVLink membership, nil where the
+	// driver reports none.
+	Fabric *GPUFabric `json:"fabric,omitempty"`
 	// Health carries anything that should keep the GPU out of placement.
 	Health GPUHealth `json:"health"`
 }
@@ -117,6 +124,9 @@ type Inventory struct {
 	// Storage is the disk behind those weights, one entry per filesystem.
 	// Empty means the same thing it means for Weights: not asked.
 	Storage []Storage `json:"storage,omitempty"`
+	// IMEX is the service a multi-node NVLink domain needs, read where the
+	// node has GPUs.
+	IMEX IMEX `json:"imex"`
 }
 
 // Classes returns the distinct GPU classes present, sorted.

internal/agent/discovery/gpu_fabric.go

@@ -0,0 +1,160 @@
+package discovery
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// GPUFabric is a GPU's place in a multi-node NVLink domain, as the driver
+// reports it: the NVL72 rack, or the NVLink Switch system, it belongs to.
+type GPUFabric struct {
+	// State is Completed once the fabric manager has joined the GPU to its
+	// domain; In Progress, Not Started or Not Supported otherwise.
+	State string `json:"state"`
+	// Status is Success, or the reason the join failed.
+	Status string `json:"status,omitempty"`
+	// ClusterUUID and CliqueID name the domain: GPUs sharing both can reach
+	// each other over NVLink whatever node they are in.
+	ClusterUUID string `json:"cluster_uuid,omitempty"`
+	CliqueID    string `json:"clique_id,omitempty"`
+	// Bandwidth is Full, or how the domain's NVLink is degraded.
+	Bandwidth string `json:"bandwidth,omitempty"`
+}
+
+// Domain is the NVLink domain the GPU is joined to, empty when it is in none
+// that crosses nodes: a card with no fabric, one not joined, or one whose
+// cluster is all zeros, which is what a single machine's NVSwitch reports.
+func (f *GPUFabric) Domain() string {
+	if f == nil || !strings.EqualFold(f.State, "Completed") {
+		return ""
+	}
+	u := strings.ToLower(strings.TrimSpace(f.ClusterUUID))
+	if u == "" || strings.Trim(u, "0-") == "" {
+		return ""
+	}
+	return u + "/" + f.CliqueID
+}
+
+// ParseGPUFabric reads the Fabric sections of `nvidia-smi -q`, by the PCI
+// address each GPU's section starts with.
+func ParseGPUFabric(out string) map[string]GPUFabric {
+	res := map[string]GPUFabric{}
+	var bus string
+	fabricIndent := -1
+	var cur *GPUFabric
+	flush := func() {
+		if bus != "" && cur != nil && cur.State != "" {
+			res[bus] = *cur
+		}
+		cur, fabricIndent = nil, -1
+	}
+	for _, line := range strings.Split(out, "\n") {
+		trim := strings.TrimSpace(line)
+		indent := len(line) - len(strings.TrimLeft(line, " \t"))
+		if strings.HasPrefix(line, "GPU ") && indent == 0 {
+			flush()
+			bus = strings.ToLower(strings.TrimSpace(strings.TrimPrefix(line, "GPU ")))
+			continue
+		}
+		if trim == "Fabric" {
+			flush()
+			cur, fabricIndent = &GPUFabric{}, indent
+			continue
+		}
+		if cur == nil {
+			continue
+		}
+		if trim != "" && indent <= fabricIndent {
+			flush()
+			continue
+		}
+		k, v, ok := strings.Cut(trim, ":")
+		if !ok {
+			continue
+		}
+		k, v = strings.TrimSpace(k), strings.TrimSpace(v)
+		if strings.EqualFold(v, "N/A") {
+			v = ""
+		}
+		switch k {
+		case "State":
+			cur.State = v
+		case "Status":
+			cur.Status = v
+		case "ClusterUUID", "Cluster UUID":
+			cur.ClusterUUID = v
+		case "CliqueId", "Clique ID":
+			cur.CliqueID = v
+		case "Bandwidth":
+			cur.Bandwidth = v
+		}
+	}
+	flush()
+	return res
+}
+
+// DiscoverGPUFabric runs `nvidia-smi -q` for the Fabric sections. Nothing
+// is reported on a driver or a card without them.
+func DiscoverGPUFabric(ctx context.Context, binary string) (map[string]GPUFabric, error) {
+	if binary == "" {
+		if _, err := exec.LookPath("nvidia-smi"); err != nil {
+			return nil, nil
+		}
+		binary = "nvidia-smi"
+	}
+	ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
+	defer cancel()
+	out, err := exec.CommandContext(ctx, binary, "-q").Output()
+	if err != nil {
+		return nil, err
+	}
+	return ParseGPUFabric(string(out)), nil
+}
+
+// ApplyFabric puts each GPU's fabric on it, matched by PCI address.
+func ApplyFabric(gpus []GPU, fab map[string]GPUFabric) {
+	for i := range gpus {
+		if f, ok := fab[strings.ToLower(gpus[i].PCIBusID)]; ok {
+			f := f
+			gpus[i].Fabric = &f
+		}
+	}
+}
+
+// IMEX is the service that lets GPUs in different nodes of one NVLink
+// domain share memory: without it, multi-node NVLink carries nothing.
+type IMEX struct {
+	// State is systemd's word for nvidia-imex: active, inactive, or empty
+	// where it is not installed.
+	State string `json:"state,omitempty"`
+	// Channels is how many IMEX channels exist; a process needs one.
+	Channels int `json:"channels"`
+}
+
+// DiscoverIMEX reads the IMEX service and its channels.
+func DiscoverIMEX(ctx context.Context) IMEX {
+	var m IMEX
+	if entries, err := os.ReadDir("/dev/nvidia-caps-imex-channels"); err == nil {
+		m.Channels = len(entries)
+	}
+	if _, err := exec.LookPath("systemctl"); err == nil {
+		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		defer cancel()
+		out, _ := exec.CommandContext(ctx, "systemctl", "is-active", "nvidia-imex").Output()
+		m.State = strings.TrimSpace(string(out))
+		if m.State == "inactive" {
+			// "inactive" is also what systemd says of a unit that does not
+			// exist; the binary says which.
+			if _, err := exec.LookPath("nvidia-imex"); err != nil {
+				if _, err := os.Stat(filepath.Join("/usr/bin", "nvidia-imex")); err != nil {
+					m.State = ""
+				}
+			}
+		}
+	}
+	return m
+}

internal/agent/discovery/gpu_fabric_test.go

@@ -0,0 +1,77 @@
+package discovery
+
+import "testing"
+
+// smiQ is an excerpt of `nvidia-smi -q` on a GB200 tray: two GPUs joined to
+// one NVL72 domain, then a PCIe card that has no fabric at all.
+const smiQ = `
+==============NVSMI LOG==============
+
+Timestamp                                 : Mon Sep 29 10:00:00 2026
+Driver Version                            : 570.86.15
+Attached GPUs                             : 3
+GPU 00000008:01:00.0
+    Product Name                          : NVIDIA GB200
+    Minor Number                          : 0
+    Fabric
+        State                             : Completed
+        Status                            : Success
+        CliqueId                          : 32766
+        ClusterUUID                       : 76B1AD76-7c24-4b6b-9f6c-2a1e0d8c11aa
+        Health
+            Bandwidth                     : Full
+    Processes                             : None
+
+GPU 00000009:01:00.0
+    Product Name                          : NVIDIA GB200
+    Fabric
+        State                             : In Progress
+        Status                            : N/A
+        CliqueId                          : N/A
+        ClusterUUID                       : N/A
+    Processes                             : None
+
+GPU 0000000A:01:00.0
+    Product Name                          : NVIDIA L40S
+    Fabric
+        State                             : N/A
+        Status                            : N/A
+`
+
+func TestAGPUsNVLinkDomainIsRead(t *testing.T) {
+	fab := ParseGPUFabric(smiQ)
+	a, ok := fab["00000008:01:00.0"]
+	if !ok {
+		t.Fatalf("fabric = %+v", fab)
+	}
+	if a.State != "Completed" || a.CliqueID != "32766" || a.Bandwidth != "Full" {
+		t.Errorf("GB200 = %+v", a)
+	}
+	if got := a.Domain(); got != "76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa/32766" {
+		t.Errorf("domain = %q", got)
+	}
+	// Still joining: no domain yet, and said as such.
+	b := fab["00000009:01:00.0"]
+	if b.State != "In Progress" || b.Domain() != "" {
+		t.Errorf("joining GPU = %+v", b)
+	}
+	// A card with no fabric reports nothing rather than an empty one.
+	if _, ok := fab["0000000a:01:00.0"]; ok {
+		t.Error("a card without a fabric was given one")
+	}
+
+	gpus := []GPU{{Index: 0, PCIBusID: "00000008:01:00.0"}, {Index: 1, PCIBusID: "0000000A:01:00.0"}}
+	ApplyFabric(gpus, fab)
+	if gpus[0].Fabric == nil || gpus[1].Fabric != nil {
+		t.Errorf("applied = %+v %+v", gpus[0].Fabric, gpus[1].Fabric)
+	}
+}
+
+func TestASingleMachinesNVSwitchIsNoDomain(t *testing.T) {
+	// An eight-GPU HGX reports its fabric joined, with a cluster of zeros: its
+	// NVLink ends at the chassis.
+	f := &GPUFabric{State: "Completed", ClusterUUID: "00000000-0000-0000-0000-000000000000", CliqueID: "0"}
+	if f.Domain() != "" {
+		t.Errorf("a single machine read as a domain: %q", f.Domain())
+	}
+}

internal/agent/discovery/gpu_smi.go

@@ -40,8 +40,8 @@ const mandatorySMIFields = 10
 //
 // This is the cgo-free path (plan §3.2): the NVML bindings need cgo, which
 // would force the whole binary — server included — to be built with a C
-// toolchain. nvidia-smi gives everything except NVLink topology, which the
-// NVML build adds.
+// toolchain. The query gives everything except topology, which Discover
+// reads from `nvidia-smi topo -m` afterwards.
 func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) {
 	if binary == "" {
 		// Not configured and not installed is a node without an NVIDIA card,
@@ -106,9 +106,27 @@ func parseSMI(out string) ([]GPU, error) {
 	return gpus, nil
 }
 
-// residentMemoryToleranceMB is how much allocated memory is written off as
+// ForeignMemoryToleranceMB is how much allocated memory is written off as
 // driver overhead rather than a foreign process.
-const residentMemoryToleranceMB = 512
+const ForeignMemoryToleranceMB = 512
+
+// foreignMemorySuffix ends the problem a card with another tenant on it is
+// kept out of placement for. One phrase, so that the control plane can find
+// it again and drop it when the tenant leaves.
+const foreignMemorySuffix = " MiB already allocated by another process"
+
+// ForeignMemoryProblem is the problem for mb of memory held by something
+// that is not one of this agent's engines, or "" when that is too little to
+// count.
+func ForeignMemoryProblem(mb int) string {
+	if mb <= ForeignMemoryToleranceMB {
+		return ""
+	}
+	return fmt.Sprintf("%d%s", mb, foreignMemorySuffix)
+}
+
+// IsForeignMemoryProblem reports whether a problem is ForeignMemoryProblem's.
+func IsForeignMemoryProblem(p string) bool { return strings.HasSuffix(p, foreignMemorySuffix) }
 
 // assessHealth flags conditions that should keep a GPU out of placement.
 func assessHealth(g *GPU) {
@@ -119,8 +137,12 @@ func assessHealth(g *GPU) {
 	// Memory already in use means something else is on the card. Starting an
 	// engine there would either fail to allocate or quietly evict the other
 	// process's throughput.
-	if g.MemoryUsedMB > residentMemoryToleranceMB {
-		g.Health.Problems = append(g.Health.Problems,
-			fmt.Sprintf("%d MiB already allocated by another process", g.MemoryUsedMB))
+	//
+	// Only a first reading: this runs before the agent knows which processes
+	// are its own, so an engine still running from before a restart reads as
+	// foreign here. Every heartbeat measures it again, without the agent's
+	// own engines, and the control plane replaces this with that.
+	if p := ForeignMemoryProblem(g.MemoryUsedMB); p != "" {
+		g.Health.Problems = append(g.Health.Problems, p)
 	}
 }

internal/agent/discovery/gpuhealth.go

@@ -0,0 +1,280 @@
+package discovery
+
+import (
+	"context"
+	"encoding/xml"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+// GPUWear is what wears a card out or takes it down, beyond temperature:
+// read every minute, from nvidia-smi and the kernel's log. The early signs of
+// a card failing are here — a growing count of rows its memory had to remap,
+// PCIe link retries, XID errors — long before a replica on it fails.
+//
+// A count of -1 is not reported: a card without row remapping (before
+// Ampere) has no such count, which is not the same as none.
+type GPUWear struct {
+	Index int
+	// PCIeReplays are link-level retries on the card's PCIe link: a few are
+	// noise, a count that keeps climbing is a riser, slot or cable.
+	PCIeReplays int64
+	// RetiredSBE and RetiredDBE are memory pages retired for single and
+	// double bit errors (cards before Ampere); RetirePending is a page
+	// waiting for a reset to be retired.
+	RetiredSBE, RetiredDBE int64
+	RetirePending          bool
+	// RemappedCorrectable and RemappedUncorrectable are memory rows remapped
+	// (Ampere and later). RemapPending waits for a reset; RemapFailure is a
+	// card out of spare rows — it needs replacing.
+	RemappedCorrectable, RemappedUncorrectable int64
+	RemapPending, RemapFailure                 bool
+	// NVLink errors summed over the card's links, by kind.
+	NVLinkCRC, NVLinkReplay, NVLinkRecovery int64
+	// XIDs are the driver's error events for this card since the agent
+	// started, and the last of them.
+	XIDs        int64
+	LastXID     int
+	LastXIDText string
+	LastXIDAt   time.Time
+}
+
+// healthEvery is how often health is read: it changes over hours, and the
+// kernel log and nvidia-smi's full report are the heavier reads.
+const healthEvery = time.Minute
+
+// HealthSampler reads GPUWear, keeping the XID counts between reads.
+type HealthSampler struct {
+	run func(ctx context.Context, name string, args ...string) ([]byte, error)
+	now func() time.Time
+
+	mu      sync.Mutex
+	at      time.Time
+	last    []GPUWear
+	since   time.Time
+	xids    map[int]*GPUWear
+	started bool
+}
+
+// NewHealthSampler reads the machine's own nvidia-smi and kernel log.
+func NewHealthSampler() *HealthSampler {
+	return &HealthSampler{now: time.Now, xids: map[int]*GPUWear{},
+		run: func(ctx context.Context, name string, args ...string) ([]byte, error) {
+			return exec.CommandContext(ctx, name, args...).Output() //nolint:gosec // fixed tools
+		}}
+}
+
+// Sample is the cards' health, read again once healthEvery has passed. Nil
+// on a machine without nvidia-smi.
+func (h *HealthSampler) Sample(ctx context.Context) []GPUWear {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	now := h.now()
+	if !h.at.IsZero() && now.Sub(h.at) < healthEvery {
+		return h.last
+	}
+	h.at = now
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+	defer cancel()
+	report, err := h.run(ctx, "nvidia-smi", "-q", "-x")
+	if err != nil {
+		h.last = nil
+		return nil
+	}
+	cards, bus := parseHealthReport(report)
+	if out, err := h.run(ctx, "nvidia-smi", "nvlink", "-e"); err == nil {
+		for i, e := range parseNVLinkErrors(string(out)) {
+			if i < len(cards) {
+				cards[i].NVLinkCRC, cards[i].NVLinkReplay, cards[i].NVLinkRecovery = e[0], e[1], e[2]
+			}
+		}
+	}
+	h.readXIDs(ctx, now, bus)
+	for i := range cards {
+		if x := h.xids[i]; x != nil {
+			cards[i].XIDs, cards[i].LastXID, cards[i].LastXIDText, cards[i].LastXIDAt = x.XIDs, x.LastXID, x.LastXIDText, x.LastXIDAt
+		}
+	}
+	h.last = cards
+	return cards
+}
+
+// readXIDs adds the XID events logged since the last read. The first read
+// starts from now: what happened before the agent started was before, and
+// counting the whole log would raise every alert on every restart.
+func (h *HealthSampler) readXIDs(ctx context.Context, now time.Time, bus map[string]int) {
+	if !h.started {
+		h.started, h.since = true, now
+		return
+	}
+	since := h.since
+	h.since = now
+	out, err := h.run(ctx, "journalctl", "-k", "--no-pager", "-o", "short-unix",
+		"--since", "@"+strconv.FormatInt(since.Unix(), 10), "-g", "NVRM: Xid")
+	if err != nil {
+		// No journal, or nothing matched (journalctl exits 1 for that).
+		return
+	}
+	for _, e := range parseXIDs(string(out)) {
+		i, ok := bus[e.bus]
+		if !ok {
+			continue
+		}
+		x := h.xids[i]
+		if x == nil {
+			x = &GPUWear{Index: i}
+			h.xids[i] = x
+		}
+		x.XIDs++
+		x.LastXID, x.LastXIDText = e.code, e.text
+		x.LastXIDAt = now
+		if e.at > 0 {
+			x.LastXIDAt = time.Unix(int64(e.at), 0)
+		}
+	}
+}
+
+// healthLog is the part of nvidia-smi's XML report health is read from.
+type healthLog struct {
+	GPUs []struct {
+		ID  string `xml:"id,attr"`
+		PCI struct {
+			Replay string `xml:"replay_counter"`
+		} `xml:"pci"`
+		Retired struct {
+			SBE     string `xml:"multiple_single_bit_retirement>retired_count"`
+			DBE     string `xml:"double_bit_retirement>retired_count"`
+			Pending string `xml:"pending_retirement"`
+			// Older drivers name it pending_blacklist.
+			Blacklist string `xml:"pending_blacklist"`
+		} `xml:"retired_pages"`
+		Remapped struct {
+			Corr    string `xml:"remapped_row_corr"`
+			Unc     string `xml:"remapped_row_unc"`
+			Pending string `xml:"remapped_row_pending"`
+			Failure string `xml:"remapped_row_failure"`
+		} `xml:"remapped_rows"`
+	} `xml:"gpu"`
+}
+
+// parseHealthReport reads each card's health, in index order, and maps each
+// card's PCI bus ("3b:00", as the kernel names it) to its index.
+func parseHealthReport(report []byte) ([]GPUWear, map[string]int) {
+	var log healthLog
+	if err := xml.Unmarshal(report, &log); err != nil {
+		return nil, nil
+	}
+	bus := map[string]int{}
+	out := make([]GPUWear, 0, len(log.GPUs))
+	for i, g := range log.GPUs {
+		bus[busOf(g.ID)] = i
+		out = append(out, GPUWear{
+			Index:                 i,
+			PCIeReplays:           count(g.PCI.Replay),
+			RetiredSBE:            count(g.Retired.SBE),
+			RetiredDBE:            count(g.Retired.DBE),
+			RetirePending:         yes(g.Retired.Pending) || yes(g.Retired.Blacklist),
+			RemappedCorrectable:   count(g.Remapped.Corr),
+			RemappedUncorrectable: count(g.Remapped.Unc),
+			RemapPending:          yes(g.Remapped.Pending),
+			RemapFailure:          yes(g.Remapped.Failure),
+		})
+	}
+	return out, bus
+}
+
+// busOf is a PCI address as the kernel's XID lines carry it: nvidia-smi
+// says "00000000:3B:00.0", the kernel "0000:3b:00".
+func busOf(id string) string {
+	id = strings.ToLower(strings.TrimSpace(id))
+	if i := strings.LastIndex(id, "."); i >= 0 {
+		id = id[:i]
+	}
+	parts := strings.Split(id, ":")
+	if len(parts) >= 2 {
+		return parts[len(parts)-2] + ":" + parts[len(parts)-1]
+	}
+	return id
+}
+
+func count(s string) int64 {
+	n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
+	if err != nil {
+		return -1
+	}
+	return n
+}
+
+func yes(s string) bool { return strings.EqualFold(strings.TrimSpace(s), "yes") }
+
+// nvlinkError matches a line of `nvidia-smi nvlink -e`, whose wording varies
+// by driver: "Link 0: Replay Errors: 0", "Link 0: CRC Errors: 0",
+// "Link 3: Data CRC Errors: 2".
+var (
+	nvlinkGPU   = regexp.MustCompile(`^GPU (\d+):`)
+	nvlinkError = regexp.MustCompile(`Link \d+:\s*(.*?)\s*Errors?\s*:\s*(\d+)`)
+)
+
+// parseNVLinkErrors sums each card's link errors by kind: CRC, replay,
+// recovery.
+func parseNVLinkErrors(out string) map[int][3]int64 {
+	res := map[int][3]int64{}
+	gpu := -1
+	for _, line := range strings.Split(out, "\n") {
+		if m := nvlinkGPU.FindStringSubmatch(strings.TrimSpace(line)); m != nil {
+			gpu, _ = strconv.Atoi(m[1])
+			continue
+		}
+		m := nvlinkError.FindStringSubmatch(line)
+		if m == nil || gpu < 0 {
+			continue
+		}
+		n, _ := strconv.ParseInt(m[2], 10, 64)
+		e := res[gpu]
+		kind := strings.ToLower(m[1])
+		switch {
+		case strings.Contains(kind, "crc"):
+			e[0] += n
+		case strings.Contains(kind, "replay"):
+			e[1] += n
+		case strings.Contains(kind, "recovery"):
+			e[2] += n
+		}
+		res[gpu] = e
+	}
+	return res
+}
+
+type xidEvent struct {
+	bus  string
+	code int
+	text string
+	at   float64
+}
+
+// xidLine matches the driver's report of an XID:
+// "NVRM: Xid (PCI:0000:3b:00): 79, pid=..., GPU has fallen off the bus."
+var xidLine = regexp.MustCompile(`NVRM: Xid \(PCI:([0-9a-fA-F:.]+)\): (\d+),\s*(.*)$`)
+
+// parseXIDs reads journalctl's short-unix lines: a timestamp, then the
+// message.
+func parseXIDs(out string) []xidEvent {
+	var evs []xidEvent
+	for _, line := range strings.Split(out, "\n") {
+		m := xidLine.FindStringSubmatch(line)
+		if m == nil {
+			continue
+		}
+		code, _ := strconv.Atoi(m[2])
+		e := xidEvent{bus: busOf(m[1]), code: code, text: strings.TrimSpace(m[3])}
+		if f := strings.Fields(line); len(f) > 0 {
+			e.at, _ = strconv.ParseFloat(f[0], 64)
+		}
+		evs = append(evs, e)
+	}
+	return evs
+}

internal/agent/discovery/gpuhealth_test.go

@@ -0,0 +1,101 @@
+package discovery
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+)
+
+const healthXML = `<?xml version="1.0" ?>
+<nvidia_smi_log>
+  <gpu id="00000000:3B:00.0">
+    <pci><replay_counter>0</replay_counter></pci>
+    <retired_pages>
+      <multiple_single_bit_retirement><retired_count>N/A</retired_count></multiple_single_bit_retirement>
+      <double_bit_retirement><retired_count>N/A</retired_count></double_bit_retirement>
+      <pending_retirement>N/A</pending_retirement>
+    </retired_pages>
+    <remapped_rows>
+      <remapped_row_corr>2</remapped_row_corr>
+      <remapped_row_unc>1</remapped_row_unc>
+      <remapped_row_pending>Yes</remapped_row_pending>
+      <remapped_row_failure>No</remapped_row_failure>
+    </remapped_rows>
+  </gpu>
+  <gpu id="00000000:5E:00.0">
+    <pci><replay_counter>4021</replay_counter></pci>
+    <retired_pages>
+      <multiple_single_bit_retirement><retired_count>3</retired_count></multiple_single_bit_retirement>
+      <double_bit_retirement><retired_count>1</retired_count></double_bit_retirement>
+      <pending_blacklist>Yes</pending_blacklist>
+    </retired_pages>
+    <remapped_rows>N/A</remapped_rows>
+  </gpu>
+</nvidia_smi_log>`
+
+const nvlinkOut = `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-aaa)
+	 Link 0: Replay Errors: 1
+	 Link 0: Recovery Errors: 0
+	 Link 0: CRC Errors: 5
+	 Link 1: Data CRC Errors: 2
+GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-bbb)
+	 Link 0: Replay Errors: 0
+`
+
+func TestAGPUsWearAndFailuresAreReadAndCountedFromWhenTheAgentStarted(t *testing.T) {
+	now := time.Unix(1_800_000_000, 0)
+	journal := ""
+	h := &HealthSampler{now: func() time.Time { return now }, xids: map[int]*GPUWear{},
+		run: func(_ context.Context, name string, args ...string) ([]byte, error) {
+			switch {
+			case name == "nvidia-smi" && args[0] == "-q":
+				return []byte(healthXML), nil
+			case name == "nvidia-smi" && args[0] == "nvlink":
+				return []byte(nvlinkOut), nil
+			case name == "journalctl":
+				if journal == "" {
+					return nil, errors.New("exit status 1")
+				}
+				return []byte(journal), nil
+			}
+			return nil, errors.New("unexpected " + name)
+		}}
+	got := h.Sample(t.Context())
+	if len(got) != 2 {
+		t.Fatalf("%+v", got)
+	}
+	a, b := got[0], got[1]
+	if a.RemappedCorrectable != 2 || a.RemappedUncorrectable != 1 || !a.RemapPending || a.RemapFailure ||
+		a.RetiredSBE != -1 || a.PCIeReplays != 0 || a.NVLinkCRC != 7 || a.NVLinkReplay != 1 {
+		t.Errorf("card 0 %+v", a)
+	}
+	if b.PCIeReplays != 4021 || b.RetiredSBE != 3 || b.RetiredDBE != 1 || !b.RetirePending || b.RemappedCorrectable != -1 {
+		t.Errorf("card 1 %+v", b)
+	}
+
+	// An XID on card 1, logged after the agent started; read on the next
+	// sample a minute on.
+	journal = "1800000030.123456 gpu07 kernel: NVRM: Xid (PCI:0000:5e:00): 79, pid=1234, name=llama-server, GPU has fallen off the bus.\n"
+	now = now.Add(61 * time.Second)
+	got = h.Sample(t.Context())
+	if got[1].XIDs != 1 || got[1].LastXID != 79 || !strings.Contains(got[1].LastXIDText, "fallen off the bus") ||
+		got[1].LastXIDAt.Unix() != 1_800_000_030 || got[0].XIDs != 0 {
+		t.Errorf("after an XID: %+v / %+v", got[0], got[1])
+	}
+	// Within the minute, the last reading.
+	journal = ""
+	now = now.Add(10 * time.Second)
+	if again := h.Sample(t.Context()); again[1].XIDs != 1 {
+		t.Errorf("%+v", again[1])
+	}
+}
+
+func TestBusIDsAreReadAsTheKernelWritesThem(t *testing.T) {
+	for in, want := range map[string]string{"00000000:3B:00.0": "3b:00", "0000:3b:00": "3b:00", "0000:5e:00.0": "5e:00"} {
+		if got := busOf(in); got != want {
+			t.Errorf("%s: %s, want %s", in, got, want)
+		}
+	}
+}

internal/agent/discovery/gpuprocs.go

@@ -0,0 +1,120 @@
+package discovery
+
+import (
+	"context"
+	"encoding/xml"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+)
+
+// GPUProcesses reports which processes hold memory on each GPU, by the GPU's
+// nvidia-smi index: what a process actually computes on, as opposed to what it
+// was told to use. A llama.cpp started with --n-gpu-layers on a GPU whose
+// backend did not load runs on the CPU without a word, and this is where that
+// shows — the process is on no GPU's list.
+//
+// From the full query rather than --query-compute-apps, which lists compute
+// processes only: llama.cpp's Vulkan build is listed as a graphics one, and
+// would have read as not on its GPU at all.
+//
+// A node with no nvidia-smi returns nil and no error, as discovery does.
+func GPUProcesses(ctx context.Context, binary string) (map[int][]int, error) {
+	if binary == "" {
+		if _, err := exec.LookPath("nvidia-smi"); err != nil {
+			return nil, nil
+		}
+		binary = "nvidia-smi"
+	}
+	out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output()
+	if err != nil {
+		return nil, fmt.Errorf("running %s: %w", binary, err)
+	}
+	return parseGPUProcesses(out)
+}
+
+// GPUMemory is one card's memory as nvidia-smi reports it at a moment: what
+// is allocated on it, and which processes hold how much of that.
+type GPUMemory struct {
+	UsedMB int
+	Procs  []GPUProcess
+}
+
+// GPUProcess is one process holding memory on a card.
+type GPUProcess struct {
+	PID    int
+	UsedMB int
+}
+
+// GPUMemoryByProcess reports each GPU's memory, by index, with the processes
+// holding it — from the same report as GPUProcesses, so the two never
+// disagree about who is on a card.
+//
+// A node with no nvidia-smi returns nil and no error, as discovery does.
+func GPUMemoryByProcess(ctx context.Context, binary string) ([]GPUMemory, error) {
+	if binary == "" {
+		if _, err := exec.LookPath("nvidia-smi"); err != nil {
+			return nil, nil
+		}
+		binary = "nvidia-smi"
+	}
+	out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output()
+	if err != nil {
+		return nil, fmt.Errorf("running %s: %w", binary, err)
+	}
+	return parseGPUMemory(out)
+}
+
+// smiLog is the part of nvidia-smi's XML report this reads: each GPU, in
+// index order, its memory and the processes on it.
+type smiLog struct {
+	GPUs []struct {
+		Used      string `xml:"fb_memory_usage>used"`
+		Processes []struct {
+			PID    string `xml:"pid"`
+			Type   string `xml:"type"`
+			UsedMB string `xml:"used_memory"`
+		} `xml:"processes>process_info"`
+	} `xml:"gpu"`
+}
+
+// parseGPUProcesses reads the report. GPUs are listed in index order, which
+// is nvidia-smi's own numbering — the one placement assigns by.
+func parseGPUProcesses(report []byte) (map[int][]int, error) {
+	mem, err := parseGPUMemory(report)
+	if err != nil {
+		return nil, err
+	}
+	out := map[int][]int{}
+	for i, g := range mem {
+		for _, p := range g.Procs {
+			out[i] = append(out[i], p.PID)
+		}
+	}
+	return out, nil
+}
+
+// parseGPUMemory reads the report into each GPU's memory and its holders.
+func parseGPUMemory(report []byte) ([]GPUMemory, error) {
+	var log smiLog
+	if err := xml.Unmarshal(report, &log); err != nil {
+		return nil, fmt.Errorf("reading nvidia-smi's report: %w", err)
+	}
+	out := make([]GPUMemory, len(log.GPUs))
+	for i, g := range log.GPUs {
+		out[i].UsedMB = mib(g.Used)
+		for _, p := range g.Processes {
+			if pid, err := strconv.Atoi(strings.TrimSpace(p.PID)); err == nil {
+				out[i].Procs = append(out[i].Procs, GPUProcess{PID: pid, UsedMB: mib(p.UsedMB)})
+			}
+		}
+	}
+	return out, nil
+}
+
+// mib reads "4932 MiB"; anything else, "N/A" included, is zero.
+func mib(s string) int {
+	n, _ := strconv.Atoi(strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(s), "MiB")))
+	return n
+}

internal/agent/discovery/gpuprocs_test.go

@@ -0,0 +1,78 @@
+package discovery
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestGPUProcessesOfEveryKindAreListedByIndex(t *testing.T) {
+	report := `<?xml version="1.0" ?>
+<nvidia_smi_log>
+  <driver_version>580.65.06</driver_version>
+  <gpu id="00000000:17:00.0"><product_name>Tesla V100-SXM2-32GB</product_name>
+    <processes></processes>
+  </gpu>
+  <gpu id="00000000:65:00.0"><product_name>Tesla V100-SXM2-32GB</product_name>
+    <processes>
+      <process_info><pid>4100</pid><type>C</type><process_name>llama-server</process_name></process_info>
+      <process_info><pid>4300</pid><type>G</type><process_name>llama-server</process_name></process_info>
+    </processes>
+  </gpu>
+  <gpu id="00000000:B3:00.0"><processes>
+      <process_info><pid>4200</pid><type>C+G</type></process_info>
+  </processes></gpu>
+</nvidia_smi_log>`
+	got, err := parseGPUProcesses([]byte(report))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// A Vulkan engine is a graphics process ("G") and is on its GPU all the
+	// same; a compute-only query would have missed it.
+	want := map[int][]int{1: {4100, 4300}, 2: {4200}}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+	if _, err := parseGPUProcesses([]byte("not xml")); err == nil {
+		t.Error("a report that does not parse was read as no processes")
+	}
+}
+
+func TestGPUMemoryIsReadPerProcess(t *testing.T) {
+	report := `<?xml version="1.0" ?>
+<nvidia_smi_log>
+  <gpu id="00000000:01:00.0"><product_name>Quadro RTX 6000</product_name>
+    <fb_memory_usage><total>24576 MiB</total><used>14514 MiB</used><free>10062 MiB</free></fb_memory_usage>
+    <processes>
+      <process_info><pid>71315</pid><type>C</type><used_memory>4932 MiB</used_memory></process_info>
+      <process_info><pid>4100</pid><type>C</type><used_memory>9500 MiB</used_memory></process_info>
+    </processes>
+  </gpu>
+  <gpu id="00000000:02:00.0">
+    <fb_memory_usage><used>N/A</used></fb_memory_usage>
+    <processes></processes>
+  </gpu>
+</nvidia_smi_log>`
+	got, err := parseGPUMemory([]byte(report))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []GPUMemory{
+		{UsedMB: 14514, Procs: []GPUProcess{{PID: 71315, UsedMB: 4932}, {PID: 4100, UsedMB: 9500}}},
+		{},
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got %+v, want %+v", got, want)
+	}
+}
+
+func TestForeignMemoryBelowTheToleranceIsNoProblem(t *testing.T) {
+	// A CUDA context that has let go of its model still holds a few hundred
+	// MiB: ComfyUI after /free held 324.
+	if p := ForeignMemoryProblem(330); p != "" {
+		t.Errorf("330 MiB is a problem: %q", p)
+	}
+	p := ForeignMemoryProblem(5014)
+	if !IsForeignMemoryProblem(p) || IsForeignMemoryProblem("2 uncorrected ECC errors") {
+		t.Errorf("%q is not recognised as the foreign memory problem, or ECC is", p)
+	}
+}

internal/agent/discovery/host.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
 )
 
@@ -26,6 +27,9 @@ type Host struct {
 	CPUFlags []string `json:"cpu_flags,omitempty"`
 	// NVIDIADriver is the loaded driver's version, empty when there is none.
 	NVIDIADriver string `json:"nvidia_driver,omitempty"`
+	// MinComputeCapability is the oldest GPU's, 0 when there are none or it
+	// is not known. Not read here but from the GPU inventory, by the agent.
+	MinComputeCapability float64 `json:"min_compute_capability,omitempty"`
 }
 
 // Family is the packaging family: "deb", "rpm", or "" when neither is known.
@@ -152,3 +156,19 @@ func nvidiaDriverFrom(path string) string {
 	}
 	return ""
 }
+
+// MinComputeCapability is the oldest GPU's compute capability, 0 when none
+// is known: an engine built for newer GPUs is refused by the oldest.
+func MinComputeCapability(gpus []GPU) float64 {
+	low := 0.0
+	for _, g := range gpus {
+		cc, err := strconv.ParseFloat(strings.TrimSpace(g.ComputeCapability), 64)
+		if err != nil || cc <= 0 {
+			continue
+		}
+		if low == 0 || cc < low {
+			low = cc
+		}
+	}
+	return low
+}

internal/agent/discovery/ib.go

@@ -5,6 +5,7 @@ package discovery
 import (
 	"errors"
 	"fmt"
+	"net"
 	"os"
 	"path/filepath"
 	"slices"
@@ -42,8 +43,66 @@ type Port struct {
 	GIDPrefix string `json:"gid_prefix"`
 	LID       int    `json:"lid"`
 	SMLID     int    `json:"sm_lid"`
+	// Counters are the port's error and traffic counters, by their sysfs
+	// names, as far as the driver has them. The data counters are converted
+	// to bytes; sysfs counts them in four-byte words.
+	Counters map[string]uint64 `json:"counters,omitempty"`
+	// RoCEv2GID is the GID table index NCCL should use on a RoCE port: the
+	// first RoCE v2 entry holding an IPv4 address, else the first RoCE v2
+	// entry. -1 on InfiniBand, and on a RoCE port with no v2 entry. NCCL's
+	// default of 0 is a RoCE v1 link-local entry on most setups, which does
+	// not route, and a job on it hangs.
+	RoCEv2GID int `json:"rocev2_gid"`
+	// PortGUID is the port's own identifier, the interface half of its GID.
+	PortGUID string `json:"port_guid,omitempty"`
+	// SM is the subnet manager this port sees as master, from sminfo. Nil on
+	// RoCE, which has none, and when infiniband-diags is not installed.
+	SM *SMInfo `json:"sm,omitempty"`
+	// Network is, on RoCE, the IP network of the RoCE v2 address: the only
+	// thing that says two RoCE ports are on the same fabric.
+	Network string `json:"network,omitempty"`
 }
 
+// SMInfo is a subnet manager as a port sees it.
+type SMInfo struct {
+	GUID     string `json:"guid"`
+	LID      int    `json:"lid"`
+	Priority int    `json:"priority"`
+	// State is SMINFO_MASTER, SMINFO_STANDBY, ...
+	State string `json:"state"`
+}
+
+// FabricKey is what identifies the physical fabric this port is on: the
+// master subnet manager on InfiniBand, since every port on one fabric sees
+// the same one and ports on separate fabrics cannot; the IP network on RoCE.
+// Empty when neither is known, and for a port that is not up.
+func (p Port) FabricKey() string {
+	if !p.Active() {
+		return ""
+	}
+	switch {
+	case p.LinkLayer == LinkEthernet && p.Network != "":
+		return "roce:" + p.Network
+	case p.SM != nil && p.SM.GUID != "":
+		return "sm:" + p.SM.GUID
+	}
+	return ""
+}
+
+// PortCounters are the counters worth watching, from each port's counters/
+// directory. The errors are the ones that say a cable, a transceiver or a
+// switch port is going bad before a job fails on it; the data counters say
+// whether the fabric is carrying anything at all.
+var PortCounters = []string{
+	"symbol_error", "link_error_recovery", "link_downed", "port_rcv_errors",
+	"port_rcv_remote_physical_errors", "port_xmit_discards",
+	"local_link_integrity_errors", "excessive_buffer_overrun_errors",
+	"port_xmit_data", "port_rcv_data",
+}
+
+// dataCounter reports whether a counter counts data in four-byte words.
+func dataCounter(name string) bool { return name == "port_xmit_data" || name == "port_rcv_data" }
+
 // Active reports whether the port can carry traffic.
 func (p Port) Active() bool { return strings.EqualFold(p.State, "ACTIVE") }
 
@@ -56,6 +115,9 @@ type HCA struct {
 	// FirmwareVersion is reported by the driver; worth capturing because
 	// NCCL behaviour differs between firmware revisions.
 	FirmwareVersion string `json:"firmware_version,omitempty"`
+	// NodeGUID identifies the adapter on its fabric: it is the name a
+	// switch's view of the fabric, and every other tool, knows it by.
+	NodeGUID string `json:"node_guid,omitempty"`
 }
 
 // ActivePorts returns the ports that can carry traffic.
@@ -102,6 +164,7 @@ func DiscoverIB(root string) ([]HCA, error) {
 func readHCA(dir, name string) (HCA, error) {
 	h := HCA{Name: name, NUMANode: -1}
 	h.FirmwareVersion = readTrimmed(filepath.Join(dir, "fw_ver"))
+	h.NodeGUID = normalizeGUID(readTrimmed(filepath.Join(dir, "node_guid")))
 
 	// device is a symlink into the PCI tree; its target's base name is the
 	// bus address, and numa_node lives beside it.
@@ -138,6 +201,30 @@ func readPort(dir string, num int) Port {
 		LID:       parseMaybeHex(readTrimmed(filepath.Join(dir, "lid"))),
 		SMLID:     parseMaybeHex(readTrimmed(filepath.Join(dir, "sm_lid"))),
 	}
+	p.RoCEv2GID = -1
+	p.PortGUID = portGUID(readTrimmed(filepath.Join(dir, "gids", "0")))
+	if p.LinkLayer == LinkEthernet {
+		p.RoCEv2GID = rocev2GID(dir)
+		if p.RoCEv2GID >= 0 {
+			idx := strconv.Itoa(p.RoCEv2GID)
+			p.Network = RoCENetwork(readTrimmed(filepath.Join(dir, "gid_attrs", "ndevs", idx)),
+				readTrimmed(filepath.Join(dir, "gids", idx)))
+		}
+	}
+	for _, name := range PortCounters {
+		raw := readTrimmed(filepath.Join(dir, "counters", name))
+		v, err := strconv.ParseUint(raw, 10, 64)
+		if err != nil {
+			continue // not every driver has every counter
+		}
+		if dataCounter(name) {
+			v *= 4
+		}
+		if p.Counters == nil {
+			p.Counters = map[string]uint64{}
+		}
+		p.Counters[name] = v
+	}
 	return p
 }
 
@@ -201,3 +288,91 @@ func readTrimmed(path string) string {
 	}
 	return strings.TrimSpace(string(b))
 }
+
+// rocev2GID finds the GID index a RoCE port's traffic should use.
+func rocev2GID(dir string) int {
+	first := -1
+	for i := 0; i < 64; i++ {
+		typ := readTrimmed(filepath.Join(dir, "gid_attrs", "types", strconv.Itoa(i)))
+		if !strings.EqualFold(typ, "RoCE v2") {
+			continue
+		}
+		gid := strings.ToLower(readTrimmed(filepath.Join(dir, "gids", strconv.Itoa(i))))
+		if strings.HasPrefix(gid, "0000:0000:0000:0000:0000:ffff:") {
+			return i
+		}
+		if first < 0 {
+			first = i
+		}
+	}
+	return first
+}
+
+// normalizeGUID writes a GUID one way: sysfs says "0c42:a103:0045:bd6c",
+// sminfo and ibnetdiscover say "0x0c42a1030045bd6c".
+func normalizeGUID(g string) string {
+	g = strings.ToLower(strings.TrimSpace(g))
+	g = strings.TrimPrefix(g, "0x")
+	g = strings.ReplaceAll(g, ":", "")
+	if g == "" || strings.Trim(g, "0") == "" {
+		return ""
+	}
+	for len(g) < 16 {
+		g = "0" + g
+	}
+	return "0x" + g
+}
+
+// NormalizeGUID is normalizeGUID for other packages.
+func NormalizeGUID(g string) string { return normalizeGUID(g) }
+
+// portGUID is the interface identifier of a GID: its last four groups.
+func portGUID(gid string) string {
+	parts := strings.Split(gid, ":")
+	if len(parts) != 8 {
+		return ""
+	}
+	return normalizeGUID(strings.Join(parts[4:], ""))
+}
+
+// RoCENetwork is the IP network an IPv4-mapped RoCE GID's address sits in,
+// read from the interface that carries it. Replaced in tests.
+var RoCENetwork = func(ndev, gid string) string {
+	ip := gidIPv4(gid)
+	if ip == nil || ndev == "" {
+		return ""
+	}
+	ifc, err := net.InterfaceByName(ndev)
+	if err != nil {
+		return ""
+	}
+	addrs, _ := ifc.Addrs()
+	for _, a := range addrs {
+		if n, ok := a.(*net.IPNet); ok && n.IP.Equal(ip) {
+			return (&net.IPNet{IP: ip.Mask(n.Mask), Mask: n.Mask}).String()
+		}
+	}
+	return ""
+}
+
+// gidIPv4 is the IPv4 address in an IPv4-mapped GID, nil otherwise.
+func gidIPv4(gid string) net.IP {
+	g := strings.ToLower(gid)
+	const mapped = "0000:0000:0000:0000:0000:ffff:"
+	if !strings.HasPrefix(g, mapped) {
+		return nil
+	}
+	hex := strings.ReplaceAll(strings.TrimPrefix(g, mapped), ":", "")
+	if len(hex) != 8 {
+		return nil
+	}
+	b := make(net.IP, 4)
+	for i := range 4 {
+		v, err := strconv.ParseUint(hex[i*2:i*2+2], 16, 8)
+		if err != nil {
+			return nil
+		}
+		b[i] = byte(v)
+	}
+	return b
+}

internal/agent/discovery/ib_test.go

@@ -31,6 +31,9 @@ func fakeSysfs(t *testing.T) string {
 	write("ib/mlx5_0/ports/1/gids/0", "FEC0:0000:0000:000A:0002:C903:00A0:1234")
 	write("ib/mlx5_0/ports/1/lid", "0x1a")
 	write("ib/mlx5_0/ports/1/sm_lid", "1")
+	write("ib/mlx5_0/ports/1/counters/symbol_error", "3")
+	write("ib/mlx5_0/ports/1/counters/link_downed", "1")
+	write("ib/mlx5_0/ports/1/counters/port_xmit_data", "1000")
 
 	// mlx5_1: two ports, the second one down.
 	write("ib/mlx5_1/ports/1/state", "4: ACTIVE")
@@ -46,7 +49,13 @@ func fakeSysfs(t *testing.T) string {
 	write("ib/mlx5_2/ports/1/state", "4: ACTIVE")
 	write("ib/mlx5_2/ports/1/rate", "100 Gb/sec (4X EDR)")
 	write("ib/mlx5_2/ports/1/link_layer", "Ethernet")
-	write("ib/mlx5_2/ports/1/gids/0", "0000:0000:0000:0000:0000:ffff:0a00:0001")
+	write("ib/mlx5_2/ports/1/gids/0", "fe80:0000:0000:0000:0000:0000:0000:0001")
+	write("ib/mlx5_2/ports/1/gid_attrs/types/0", "IB/RoCE v1")
+	write("ib/mlx5_2/ports/1/gids/1", "fe80:0000:0000:0000:0000:0000:0000:0001")
+	write("ib/mlx5_2/ports/1/gid_attrs/types/1", "RoCE v2")
+	write("ib/mlx5_2/ports/1/gids/3", "0000:0000:0000:0000:0000:ffff:0a00:0001")
+	write("ib/mlx5_2/ports/1/gid_attrs/types/3", "RoCE v2")
+	write("ib/mlx5_2/ports/1/gid_attrs/ndevs/3", "ens1f0")
 
 	// A PCI device with a NUMA node, linked the way the driver does it.
 	pci := filepath.Join(root, "pci", "0000:0b:00.0")
@@ -214,3 +223,131 @@ func TestParseHelpers(t *testing.T) {
 		}
 	})
 }
+
+func TestAPortsCountersAreRead(t *testing.T) {
+	hcas, err := DiscoverIB(fakeSysfs(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := hcas[0].Ports[0].Counters
+	// Data in bytes: sysfs counts four-byte words.
+	if c["symbol_error"] != 3 || c["link_downed"] != 1 || c["port_xmit_data"] != 4000 {
+		t.Errorf("counters = %v", c)
+	}
+	if _, ok := c["port_rcv_errors"]; ok {
+		t.Error("a counter the driver does not have should be absent, not zero")
+	}
+}
+
+func TestRoCEFindsItsRoutableGID(t *testing.T) {
+	// NCCL's default GID index is a v1 link-local entry on most hosts, which
+	// does not route: the job hangs. The v2 entry holding an IPv4 address is
+	// the one to use, over an earlier v2 link-local one.
+	hcas, err := DiscoverIB(fakeSysfs(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, h := range hcas {
+		p := h.Ports[0]
+		switch h.Name {
+		case "mlx5_2":
+			if p.RoCEv2GID != 3 {
+				t.Errorf("RoCE GID index = %d, want 3", p.RoCEv2GID)
+			}
+		default:
+			if p.RoCEv2GID != -1 {
+				t.Errorf("%s is InfiniBand and has a GID index %d", h.Name, p.RoCEv2GID)
+			}
+		}
+	}
+}
+
+func TestTheSubnetManagerAPortSeesIsRead(t *testing.T) {
+	sm, err := ParseSMInfo("sminfo: sm lid 1 sm guid 0x98039B03009fcfd6, activity count 5452 priority 15 state 3 SMINFO_MASTER\n")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if sm.GUID != "0x98039b03009fcfd6" || sm.LID != 1 || sm.Priority != 15 || sm.State != "SMINFO_MASTER" {
+		t.Errorf("sm = %+v", sm)
+	}
+	if _, err := ParseSMInfo("ibwarn: [1234] mad_rpc_open_port: can't open UMAD port"); err == nil {
+		t.Error("an error from sminfo should not parse")
+	}
+}
+
+func TestAPortsFabricIsItsSubnetManagerOrItsNetwork(t *testing.T) {
+	// Two ports on one InfiniBand fabric see one master SM, and on separate
+	// fabrics cannot; RoCE has no SM, and is its IP network.
+	hcas, err := DiscoverIB(fakeSysfs(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if hcas[0].NodeGUID != "" {
+		t.Errorf("no node_guid file, yet %q", hcas[0].NodeGUID)
+	}
+	err = AttachSM(hcas, func(hca string, port int) (*SMInfo, error) {
+		return &SMInfo{GUID: "0x00000000000000aa", LID: 1, State: "SMINFO_MASTER"}, nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := hcas[0].Ports[0].FabricKey(); got != "sm:0x00000000000000aa" {
+		t.Errorf("IB key = %q", got)
+	}
+	if got := hcas[1].Ports[1].FabricKey(); got != "" {
+		t.Errorf("a port that is down is on no fabric, got %q", got)
+	}
+	if hcas[2].Ports[0].SM != nil {
+		t.Error("RoCE was asked for a subnet manager")
+	}
+	if got := hcas[0].Ports[0].PortGUID; got != "0x0002c90300a01234" {
+		t.Errorf("port GUID = %q", got)
+	}
+}
+
+func TestARoCEPortsNetworkIsItsFabric(t *testing.T) {
+	old := RoCENetwork
+	t.Cleanup(func() { RoCENetwork = old })
+	RoCENetwork = func(ndev, gid string) string {
+		if ip := gidIPv4(gid); ip != nil && ip.String() == "10.0.0.1" {
+			return "10.0.0.0/24"
+		}
+		return ""
+	}
+	hcas, err := DiscoverIB(fakeSysfs(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	roce := hcas[2].Ports[0]
+	if roce.Network != "10.0.0.0/24" || roce.FabricKey() != "roce:10.0.0.0/24" {
+		t.Errorf("RoCE network %q key %q", roce.Network, roce.FabricKey())
+	}
+}
+
+// smirDump is `saquery SMIR` on a fabric with a master and a standby, in the
+// layout infiniband-diags prints.
+const smirDump = "SMInfoRecord dump:\n\t\tRID\n\t\tLID...................1\n\t\tSMInfo dump:\n" +
+	"\t\tGUID..................0x248a07030030f0c0\n\t\tSM_Key................0x0000000000000000\n" +
+	"\t\tActCount..............386773\n\t\tPriority..............15\n\t\tSMState...............3\n" +
+	"SMInfoRecord dump:\n\t\tRID\n\t\tLID...................12\n\t\tSMInfo dump:\n" +
+	"\t\tGUID..................0x248a07030030f1d4\n\t\tSM_Key................0x0000000000000000\n" +
+	"\t\tActCount..............12\n\t\tPriority..............14\n\t\tSMState...............2\n"
+
+func TestEverySubnetManagerOnAFabricIsListed(t *testing.T) {
+	recs, err := ParseSMIR(smirDump)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(recs) != 2 {
+		t.Fatalf("records = %+v", recs)
+	}
+	if recs[0].GUID != "0x248a07030030f0c0" || recs[0].State != "master" || recs[0].Priority != 15 || recs[0].LID != 1 {
+		t.Errorf("master = %+v", recs[0])
+	}
+	if recs[1].State != "standby" || recs[1].LID != 12 {
+		t.Errorf("standby = %+v", recs[1])
+	}
+	if _, err := ParseSMIR("ibwarn: can't open UMAD port"); err == nil {
+		t.Error("an error should not parse")
+	}
+}

internal/agent/discovery/load.go

@@ -0,0 +1,260 @@
+package discovery
+
+import (
+	"bufio"
+	"bytes"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Load is how busy a node is: the machine as a whole, and each engine's share.
+type Load struct {
+	CPUPercent float64
+	CPUCount   int
+	Load1      float64
+	Load5      float64
+	Load15     float64
+	MemUsed    int64
+	MemAvail   int64
+	Shards     []ShardLoad
+}
+
+// ShardLoad is one engine process group's use.
+type ShardLoad struct {
+	ID       string
+	CPUCores float64
+	RSSBytes int64
+}
+
+// userHZ is the unit /proc counts CPU time in. Fixed at 100 by the kernel's
+// ABI on every architecture a node runs, whatever the internal tick rate.
+const userHZ = 100
+
+// LoadSampler measures load as the difference between two readings of /proc.
+//
+// CPU time is a counter, so a percentage needs two samples and the time
+// between them. The sampler keeps the previous one; the first call has
+// nothing to compare with and reports no load at all, rather than a zero that
+// would read as an idle machine.
+//
+// Linux only, like every node. Elsewhere the files are missing and it reports
+// nothing, as an older agent does.
+type LoadSampler struct {
+	root string
+	now  func() time.Time
+
+	mu      sync.Mutex
+	prevAt  time.Time
+	prevCPU cpuTimes
+	prevPG  map[int]groupUse // process group -> what it has used
+}
+
+// groupUse is a process group's CPU time so far and its resident memory now.
+type groupUse struct {
+	jiffies  uint64
+	rssPages int64
+}
+
+type cpuTimes struct{ total, idle uint64 }
+
+// NewLoadSampler reads the real /proc.
+func NewLoadSampler() *LoadSampler { return newLoadSampler("/proc", time.Now) }
+
+func newLoadSampler(root string, now func() time.Time) *LoadSampler {
+	return &LoadSampler{root: root, now: now}
+}
+
+// Sample reads the machine and the process groups led by the given shards'
+// PIDs, keyed by shard ID. False when there is nothing to report yet.
+func (s *LoadSampler) Sample(shards map[string]int) (Load, bool) {
+	cpu, ok := s.cpu()
+	if !ok {
+		return Load{}, false
+	}
+	groups := s.groups()
+	at := s.now()
+
+	s.mu.Lock()
+	prevAt, prevCPU, prevPG := s.prevAt, s.prevCPU, s.prevPG
+	s.prevAt, s.prevCPU, s.prevPG = at, cpu, groups
+	s.mu.Unlock()
+
+	if prevAt.IsZero() || cpu.total <= prevCPU.total {
+		return Load{}, false
+	}
+	l := Load{CPUCount: s.cpuCount()}
+	dTotal := float64(cpu.total - prevCPU.total)
+	dIdle := float64(cpu.idle - prevCPU.idle)
+	l.CPUPercent = clamp((dTotal-dIdle)/dTotal*100, 0, 100)
+	l.Load1, l.Load5, l.Load15 = s.loadavg()
+	total, avail := s.meminfo()
+	if total > 0 {
+		l.MemUsed, l.MemAvail = total-avail, avail
+	}
+
+	secs := at.Sub(prevAt).Seconds()
+	for id, pid := range shards {
+		if pid <= 0 {
+			continue
+		}
+		now := groups[pid]
+		sl := ShardLoad{ID: id, RSSBytes: now.rssPages * int64(os.Getpagesize())}
+		if before, ok := prevPG[pid]; ok && secs > 0 && now.jiffies >= before.jiffies {
+			sl.CPUCores = float64(now.jiffies-before.jiffies) / userHZ / secs
+		}
+		l.Shards = append(l.Shards, sl)
+	}
+	return l, true
+}
+
+func clamp(v, lo, hi float64) float64 { return max(lo, min(v, hi)) }
+
+// cpu reads the aggregate line of /proc/stat. Idle includes iowait: a core
+// waiting on the disk is not doing work.
+func (s *LoadSampler) cpu() (cpuTimes, bool) {
+	b, err := os.ReadFile(filepath.Join(s.root, "stat"))
+	if err != nil {
+		return cpuTimes{}, false
+	}
+	line, _, _ := bytes.Cut(b, []byte("\n"))
+	f := strings.Fields(string(line))
+	if len(f) < 5 || f[0] != "cpu" {
+		return cpuTimes{}, false
+	}
+	var t cpuTimes
+	for i, v := range f[1:] {
+		// guest and guest_nice are already counted in user and nice.
+		if i >= 8 {
+			break
+		}
+		n, _ := strconv.ParseUint(v, 10, 64)
+		t.total += n
+		if i == 3 || i == 4 {
+			t.idle += n
+		}
+	}
+	return t, true
+}
+
+func (s *LoadSampler) cpuCount() int {
+	b, err := os.ReadFile(filepath.Join(s.root, "stat"))
+	if err != nil {
+		return 0
+	}
+	n := 0
+	for _, line := range strings.Split(string(b), "\n") {
+		if strings.HasPrefix(line, "cpu") && len(line) > 3 && line[3] >= '0' && line[3] <= '9' {
+			n++
+		}
+	}
+	return n
+}
+
+func (s *LoadSampler) loadavg() (l1, l5, l15 float64) {
+	b, err := os.ReadFile(filepath.Join(s.root, "loadavg"))
+	if err != nil {
+		return 0, 0, 0
+	}
+	f := strings.Fields(string(b))
+	if len(f) < 3 {
+		return 0, 0, 0
+	}
+	l1, _ = strconv.ParseFloat(f[0], 64)
+	l5, _ = strconv.ParseFloat(f[1], 64)
+	l15, _ = strconv.ParseFloat(f[2], 64)
+	return l1, l5, l15
+}
+
+// meminfo returns MemTotal and MemAvailable in bytes.
+func (s *LoadSampler) meminfo() (total, avail int64) {
+	f, err := os.Open(filepath.Join(s.root, "meminfo"))
+	if err != nil {
+		return 0, 0
+	}
+	defer f.Close()
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		k, v, ok := strings.Cut(sc.Text(), ":")
+		if !ok {
+			continue
+		}
+		fields := strings.Fields(v)
+		if len(fields) == 0 {
+			continue
+		}
+		kb, _ := strconv.ParseInt(fields[0], 10, 64)
+		switch k {
+		case "MemTotal":
+			total = kb * 1024
+		case "MemAvailable":
+			avail = kb * 1024
+		}
+	}
+	return total, avail
+}
+
+// procStat is what one /proc/<pid>/stat says that load needs.
+type procStat struct {
+	pgrp     int
+	jiffies  uint64
+	rssPages int64
+}
+
+// readProcStat parses /proc/<pid>/stat. The command name is in parentheses
+// and may itself hold spaces and parentheses, so fields are counted from the
+// last closing one.
+func readProcStat(path string) (procStat, bool) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return procStat{}, false
+	}
+	i := bytes.LastIndexByte(b, ')')
+	if i < 0 {
+		return procStat{}, false
+	}
+	f := strings.Fields(string(b[i+1:]))
+	// After the name: state(0) ppid(1) pgrp(2) ... utime(11) stime(12) ... rss(21)
+	if len(f) < 22 {
+		return procStat{}, false
+	}
+	var st procStat
+	st.pgrp, _ = strconv.Atoi(f[2])
+	ut, _ := strconv.ParseUint(f[11], 10, 64)
+	stt, _ := strconv.ParseUint(f[12], 10, 64)
+	st.jiffies = ut + stt
+	st.rssPages, _ = strconv.ParseInt(f[21], 10, 64)
+	return st, true
+}
+
+// groups totals CPU time and memory per process group, in one pass over
+// /proc. An engine is started as the leader of its own group, so its workers
+// — vLLM starts several — are counted with it.
+func (s *LoadSampler) groups() map[int]groupUse {
+	out := map[int]groupUse{}
+	s.eachProc(func(st procStat) {
+		g := out[st.pgrp]
+		g.jiffies += st.jiffies
+		g.rssPages += st.rssPages
+		out[st.pgrp] = g
+	})
+	return out
+}
+
+func (s *LoadSampler) eachProc(fn func(procStat)) {
+	entries, err := os.ReadDir(s.root)
+	if err != nil {
+		return
+	}
+	for _, e := range entries {
+		if _, err := strconv.Atoi(e.Name()); err != nil {
+			continue
+		}
+		if st, ok := readProcStat(filepath.Join(s.root, e.Name(), "stat")); ok {
+			fn(st)
+		}
+	}
+}

internal/agent/discovery/load_test.go

@@ -0,0 +1,96 @@
+package discovery
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// fakeProc is a /proc with two CPUs, a memory picture and processes.
+type fakeProc struct {
+	t    *testing.T
+	root string
+}
+
+func newFakeProc(t *testing.T) *fakeProc {
+	f := &fakeProc{t: t, root: t.TempDir()}
+	f.write("loadavg", "1.50 0.75 0.25 2/300 4242\n")
+	f.write("meminfo", "MemTotal:       8000000 kB\nMemFree:        1000000 kB\nMemAvailable:   3000000 kB\n")
+	return f
+}
+
+func (f *fakeProc) write(name, body string) {
+	f.t.Helper()
+	p := filepath.Join(f.root, name)
+	if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+		f.t.Fatal(err)
+	}
+	if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+		f.t.Fatal(err)
+	}
+}
+
+// cpu sets the aggregate counters: user and idle, the rest zero.
+func (f *fakeProc) cpu(user, idle uint64) {
+	f.write("stat", fmt.Sprintf("cpu  %d 0 0 %d 0 0 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\nintr 1\n", user, idle))
+}
+
+// proc sets one process: its group, CPU jiffies (as utime) and RSS in pages.
+func (f *fakeProc) proc(pid, pgrp int, jiffies uint64, rss int64) {
+	// The name holds a space and a parenthesis, as real ones can.
+	f.write(fmt.Sprintf("%d/stat", pid), fmt.Sprintf(
+		"%d (llama-server (x)) S 1 %d %d 0 -1 0 0 0 0 0 %d 0 0 0 20 0 1 0 100 1000 %d 0\n",
+		pid, pgrp, pgrp, jiffies, rss))
+}
+
+func TestLoadIsTheDifferenceBetweenTwoReadings(t *testing.T) {
+	p := newFakeProc(t)
+	at := time.Unix(1000, 0)
+	s := newLoadSampler(p.root, func() time.Time { return at })
+
+	p.cpu(100, 900)
+	p.proc(42, 42, 1000, 10)
+	p.proc(43, 42, 500, 5) // a worker in the engine's group
+	p.proc(7, 7, 99999, 1) // something else entirely
+	if _, ok := s.Sample(map[string]int{"shard-a": 42}); ok {
+		t.Fatal("the first reading has nothing to compare with and should report nothing")
+	}
+
+	// Two seconds later: 300 of the 400 jiffies that passed were busy, and
+	// the engine's group used 400 of them.
+	at = at.Add(2 * time.Second)
+	p.cpu(400, 1000)
+	p.proc(42, 42, 1300, 10)
+	p.proc(43, 42, 600, 6)
+	l, ok := s.Sample(map[string]int{"shard-a": 42})
+	if !ok {
+		t.Fatal("the second reading should report")
+	}
+	if l.CPUPercent != 75 || l.CPUCount != 2 {
+		t.Errorf("cpu %.1f%% of %d, want 75%% of 2", l.CPUPercent, l.CPUCount)
+	}
+	if l.Load1 != 1.5 || l.Load15 != 0.25 {
+		t.Errorf("load %v %v", l.Load1, l.Load15)
+	}
+	if l.MemUsed != 5000000*1024 || l.MemAvail != 3000000*1024 {
+		t.Errorf("memory used %d available %d", l.MemUsed, l.MemAvail)
+	}
+	if len(l.Shards) != 1 {
+		t.Fatalf("shards = %+v", l.Shards)
+	}
+	sh := l.Shards[0]
+	// 400 jiffies over 2 seconds at 100 a second is two cores.
+	if sh.ID != "shard-a" || sh.CPUCores != 2 || sh.RSSBytes != 16*int64(os.Getpagesize()) {
+		t.Errorf("shard = %+v", sh)
+	}
+}
+
+func TestNoProcMeansNoLoad(t *testing.T) {
+	s := newLoadSampler(filepath.Join(t.TempDir(), "missing"), time.Now)
+	s.Sample(nil)
+	if _, ok := s.Sample(nil); ok {
+		t.Error("a system without /proc should report nothing, not zeros")
+	}
+}

internal/agent/discovery/perftest.go

@@ -0,0 +1,133 @@
+package discovery
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// WriteBW is one side of an ib_write_bw run, from perftest: an RDMA write
+// bandwidth test between two adapters, which is what a collective spanning
+// nodes rides on.
+type WriteBW struct {
+	HCA  string
+	Port int
+	// GIDIndex is the RoCE v2 GID to use; negative on InfiniBand, where the
+	// port's LID is used.
+	GIDIndex int
+	// Peer is the server's address, for the client; empty makes the server.
+	Peer    string
+	TCPPort int
+	Seconds int
+}
+
+// Args are the command line: a duration rather than a count, so the test
+// takes as long on a slow link as a fast one, and results in Gb/s.
+func (w WriteBW) Args() []string {
+	args := []string{"-d", w.HCA, "-i", strconv.Itoa(max(w.Port, 1)),
+		"-p", strconv.Itoa(w.TCPPort), "-D", strconv.Itoa(max(w.Seconds, 1)),
+		"-F", "--report_gbits"}
+	if w.GIDIndex >= 0 {
+		args = append(args, "-x", strconv.Itoa(w.GIDIndex))
+	}
+	if w.Peer != "" {
+		args = append(args, w.Peer)
+	}
+	return args
+}
+
+// RunWriteBW runs one side and returns the average bandwidth the output
+// reports, in Gb/s, and the output. The client retries its connection for a
+// while: the server is started first, but a few seconds may pass before it
+// listens.
+func RunWriteBW(ctx context.Context, w WriteBW) (float64, string, error) {
+	bin, err := exec.LookPath("ib_write_bw")
+	if err != nil {
+		return 0, "", errors.New("ib_write_bw is not installed (the perftest package)")
+	}
+	limit := time.Duration(w.Seconds+60) * time.Second
+	var out []byte
+	for attempt := 0; ; attempt++ {
+		cctx, cancel := context.WithTimeout(ctx, limit)
+		out, err = exec.CommandContext(cctx, bin, w.Args()...).CombinedOutput() //nolint:gosec // fixed tool
+		cancel()
+		if err == nil || w.Peer == "" || !refused(string(out)) || attempt >= 10 {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			return 0, string(out), ctx.Err()
+		case <-time.After(time.Second):
+		}
+	}
+	text := string(out)
+	if err != nil {
+		return 0, text, fmt.Errorf("ib_write_bw: %v: %s", err, lastNonEmpty(text))
+	}
+	gbps, perr := ParseWriteBW(text)
+	if perr != nil && w.Peer != "" {
+		// Only the client is sure to print the result; a server that does not
+		// is not a failure.
+		return 0, text, perr
+	}
+	return gbps, text, nil
+}
+
+func refused(out string) bool {
+	for _, s := range []string{"Couldn't connect", "Unable to init the socket connection", "Connection refused"} {
+		if strings.Contains(out, s) {
+			return true
+		}
+	}
+	return false
+}
+
+var columnSplit = regexp.MustCompile(`\s{2,}`)
+
+// ParseWriteBW finds the average bandwidth in perftest's result table:
+//
+//	#bytes     #iterations    BW peak[Gb/sec]    BW average[Gb/sec]   MsgRate[Mpps]
+//	65536      887036           0.00               96.93               0.184877
+//
+// by the heading's position rather than a fixed column, since versions add
+// columns and the peak reads 0.00 when the test runs for a duration.
+func ParseWriteBW(out string) (float64, error) {
+	lines := strings.Split(out, "\n")
+	for i, l := range lines {
+		if !strings.Contains(l, "BW average") {
+			continue
+		}
+		heads := columnSplit.Split(strings.TrimSpace(l), -1)
+		col := -1
+		for j, h := range heads {
+			if strings.HasPrefix(h, "BW average") {
+				col = j
+			}
+		}
+		for _, d := range lines[i+1:] {
+			f := strings.Fields(d)
+			if len(f) == 0 || strings.HasPrefix(f[0], "-") {
+				continue
+			}
+			if len(f) != len(heads) || col < 0 {
+				return 0, fmt.Errorf("perftest's result table has a shape this does not know: %q / %q", l, d)
+			}
+			v, err := strconv.ParseFloat(f[col], 64)
+			if err != nil {
+				return 0, fmt.Errorf("reading the average bandwidth %q: %w", f[col], err)
+			}
+			return v, nil
+		}
+	}
+	return 0, errors.New("perftest printed no result")
+}
+
+func lastNonEmpty(s string) string {
+	lines := strings.Split(strings.TrimSpace(s), "\n")
+	return strings.TrimSpace(lines[len(lines)-1])
+}

internal/agent/discovery/perftest_test.go

@@ -0,0 +1,49 @@
+package discovery
+
+import (
+	"slices"
+	"testing"
+)
+
+// The client's output of perftest 24.x, with a duration.
+const writeBWClient = `---------------------------------------------------------------------------------------
+                    RDMA_Write BW Test
+ Dual-port       : OFF		Device         : mlx5_0
+ Number of qps   : 1		Transport type : IB
+ Connection type : RC		Using SRQ      : OFF
+ TX depth        : 128
+ CQ Moderation   : 100
+ Mtu             : 4096[B]
+ Link type       : IB
+ Max inline data : 0[B]
+ rdma_cm QPs	 : OFF
+ Data ex. method : Ethernet
+---------------------------------------------------------------------------------------
+ local address: LID 0x05 QPN 0x0112 PSN 0x7b1f02 RKey 0x1fffbd VAddr 0x007f1d2a5f0000
+ remote address: LID 0x09 QPN 0x0113 PSN 0x2c1e44 RKey 0x1ffeaf VAddr 0x007fa4e1a60000
+---------------------------------------------------------------------------------------
+ #bytes     #iterations    BW peak[Gb/sec]    BW average[Gb/sec]   MsgRate[Mpps]
+ 65536      1851214          0.00               388.21		   0.740422
+---------------------------------------------------------------------------------------
+`
+
+func TestTheAverageBandwidthIsReadFromPerftest(t *testing.T) {
+	got, err := ParseWriteBW(writeBWClient)
+	if err != nil || got != 388.21 {
+		t.Errorf("got %v, %v", got, err)
+	}
+	if _, err := ParseWriteBW(" Couldn't connect to 10.0.0.2:18515\n"); err == nil {
+		t.Error("output with no result table gave a number")
+	}
+}
+
+func TestWriteBWArguments(t *testing.T) {
+	ib := WriteBW{HCA: "mlx5_0", Port: 1, GIDIndex: -1, TCPPort: 18515, Seconds: 3}.Args()
+	if slices.Contains(ib, "-x") {
+		t.Errorf("InfiniBand was given a GID index: %v", ib)
+	}
+	roce := WriteBW{HCA: "mlx5_0", Port: 1, GIDIndex: 3, Peer: "10.0.0.2", TCPPort: 18515, Seconds: 3}.Args()
+	if !slices.Contains(roce, "-x") || roce[len(roce)-1] != "10.0.0.2" {
+		t.Errorf("RoCE client: %v", roce)
+	}
+}

internal/agent/discovery/sensors.go

@@ -0,0 +1,287 @@
+package discovery
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Sensors is what a node's own hardware says about heat and power: the CPUs,
+// the board where it has an inlet sensor, and each GPU. A zero is not
+// measured, never a reading of zero — a VM has none of these, and must not
+// look like a machine running at 0 °C on no power.
+type Sensors struct {
+	// CPUTempC is the hottest CPU sensor: the package, or the hottest core
+	// where no package reading exists.
+	CPUTempC float64
+	// CPUPowerW is the CPU packages' draw, from RAPL: the change in their
+	// energy counters over the time between two samples.
+	CPUPowerW float64
+	// InletTempC is the air coming in, where the board has a sensor it calls
+	// that. Most do not, and a CPU's temperature is not a substitute: it
+	// follows the load, not the room.
+	InletTempC float64
+	GPUs       []GPUSensor
+}
+
+// GPUSensor is one card's heat, power and whether it is slowing itself down.
+type GPUSensor struct {
+	Index       int
+	TempC       float64
+	MemoryTempC float64
+	PowerW      float64
+	PowerLimitW float64
+	// Throttle names what is holding the card's clocks down now:
+	// sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake. Empty
+	// is running free. Idle and application clock settings are left out;
+	// they are not the card being held back.
+	Throttle []string
+	// How busy it is, read with the rest every sample. Discovery reads the
+	// same once, at start, which made a busy card look as idle as it was
+	// when the agent came up. -1 is not reported by the driver; zero
+	// MemoryTotalMB is a sample without these at all.
+	UtilPercent, MemUtilPercent float64
+	MemoryUsedMB, MemoryTotalMB float64
+	SMClockMHz                  float64
+	ECCUncorrected              float64
+}
+
+// sensorEvery bounds how often nvidia-smi is run. Heartbeats come more often
+// than temperatures change, and each run costs a few hundred milliseconds.
+const sensorEvery = 10 * time.Second
+
+// SensorSampler reads Sensors. The first sample has no CPU power, which needs
+// two readings of the energy counters.
+type SensorSampler struct {
+	sys string // /sys, or a test's copy of it
+	smi func(ctx context.Context, fields string) ([]byte, error)
+	now func() time.Time
+
+	mu         sync.Mutex
+	last       Sensors
+	lastAt     time.Time
+	prevEnergy map[string]uint64
+	prevAt     time.Time
+}
+
+// NewSensorSampler reads this machine.
+func NewSensorSampler() *SensorSampler {
+	return &SensorSampler{sys: "/sys", smi: runSMI, now: time.Now}
+}
+
+func runSMI(ctx context.Context, fields string) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+	//nolint:gosec // fixed binary, fields from this file
+	return exec.CommandContext(ctx, "nvidia-smi", "--query-gpu="+fields,
+		"--format=csv,noheader,nounits").Output()
+}
+
+// Sample reads the sensors, or returns the last reading when that is recent.
+func (s *SensorSampler) Sample(ctx context.Context) Sensors {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	now := s.now()
+	if !s.lastAt.IsZero() && now.Sub(s.lastAt) < sensorEvery {
+		return s.last
+	}
+	out := Sensors{}
+	out.CPUTempC, out.InletTempC = s.hwmon()
+	out.CPUPowerW = s.rapl(now)
+	out.GPUs = s.gpus(ctx)
+	s.last, s.lastAt = out, now
+	return out
+}
+
+// hwmon reads the hottest CPU sensor and an inlet sensor, from every hwmon
+// device the kernel exposes — what lm_sensors reads, without it installed.
+func (s *SensorSampler) hwmon() (cpu, inlet float64) {
+	devs, _ := filepath.Glob(filepath.Join(s.sys, "class/hwmon/hwmon*"))
+	var pkg, core float64
+	for _, d := range devs {
+		name := readTrim(filepath.Join(d, "name"))
+		inputs, _ := filepath.Glob(filepath.Join(d, "temp*_input"))
+		for _, in := range inputs {
+			milli, err := strconv.ParseFloat(readTrim(in), 64)
+			if err != nil || milli <= 0 {
+				continue
+			}
+			c := milli / 1000
+			label := strings.ToLower(readTrim(strings.TrimSuffix(in, "_input") + "_label"))
+			switch {
+			case isCPUSensor(name) && (strings.HasPrefix(label, "package") || label == "tctl" || label == "tdie"):
+				pkg = max(pkg, c)
+			case isCPUSensor(name):
+				core = max(core, c)
+			case label == "cpu":
+				// A board's own reading of the CPU, as Dell's SMM gives.
+				core = max(core, c)
+			case strings.Contains(label, "inlet") || strings.Contains(label, "ambient") ||
+				strings.Contains(label, "intake") || label == "systin":
+				inlet = max(inlet, c)
+			}
+		}
+	}
+	if pkg > 0 {
+		return pkg, inlet
+	}
+	return core, inlet
+}
+
+func isCPUSensor(name string) bool {
+	switch name {
+	case "coretemp", "k10temp", "zenpower", "cpu_thermal":
+		return true
+	}
+	return false
+}
+
+// rapl is the CPU packages' power from their energy counters. Package
+// domains only: the dram and core subdomains are parts of a package, and
+// adding them would count the same energy twice.
+func (s *SensorSampler) rapl(now time.Time) float64 {
+	doms, _ := filepath.Glob(filepath.Join(s.sys, "class/powercap/intel-rapl:*"))
+	energy := map[string]uint64{}
+	wrap := map[string]uint64{}
+	for _, d := range doms {
+		if strings.Count(filepath.Base(d), ":") != 1 {
+			continue // intel-rapl:0:0 is a subdomain of intel-rapl:0
+		}
+		e, err := strconv.ParseUint(readTrim(filepath.Join(d, "energy_uj")), 10, 64)
+		if err != nil {
+			continue // unreadable without root on newer kernels
+		}
+		energy[d] = e
+		wrap[d], _ = strconv.ParseUint(readTrim(filepath.Join(d, "max_energy_range_uj")), 10, 64)
+	}
+	prev, prevAt := s.prevEnergy, s.prevAt
+	s.prevEnergy, s.prevAt = energy, now
+	dt := now.Sub(prevAt).Seconds()
+	if prev == nil || dt <= 0 || len(energy) == 0 {
+		return 0
+	}
+	var joules float64
+	for d, e := range energy {
+		p, ok := prev[d]
+		if !ok {
+			return 0 // a domain appeared: no fair difference this time
+		}
+		if e < p { // the counter wrapped
+			e += wrap[d]
+		}
+		joules += float64(e-p) / 1e6
+	}
+	return joules / dt
+}
+
+// The throttle reasons nvidia-smi reports, as bits. Only those that mean the
+// card is being held back are named.
+var throttleBits = []struct {
+	bit  uint64
+	name string
+}{
+	{0x4, "sw_power_cap"},
+	{0x8, "hw_slowdown"},
+	{0x20, "sw_thermal"},
+	{0x40, "hw_thermal"},
+	{0x80, "power_brake"},
+}
+
+// gpus asks nvidia-smi for each card's heat and power. The memory temperature
+// is not known to every driver, and a field nvidia-smi does not know fails the
+// whole query, so it is asked for first and dropped on failure.
+// gpuFields are what each sample asks nvidia-smi for, in order. The busy
+// figures follow the throttle reasons; the memory temperature is last, being
+// the one field a driver may reject outright.
+const (
+	gpuBase  = "index,temperature.gpu,power.draw,enforced.power.limit,clocks_throttle_reasons.active"
+	gpuBusy  = ",utilization.gpu,utilization.memory,memory.used,memory.total,clocks.sm,ecc.errors.uncorrected.volatile.total"
+	gpuHBMTC = ",temperature.memory"
+)
+
+func (s *SensorSampler) gpus(ctx context.Context) []GPUSensor {
+	// Most fields first, fewer on a driver that refuses one: a query naming
+	// a field the driver does not know fails as a whole.
+	var out []byte
+	var busy bool
+	for _, q := range []struct {
+		fields string
+		busy   bool
+	}{{gpuBase + gpuBusy + gpuHBMTC, true}, {gpuBase + gpuBusy, true}, {gpuBase + gpuHBMTC, false}, {gpuBase, false}} {
+		var err error
+		if out, err = s.smi(ctx, q.fields); err == nil {
+			busy = q.busy
+			break
+		}
+		out = nil
+	}
+	if out == nil {
+		return nil
+	}
+	var gs []GPUSensor
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		f := strings.Split(line, ",")
+		if len(f) < 5 {
+			continue
+		}
+		for i := range f {
+			f[i] = strings.TrimSpace(f[i])
+		}
+		idx, err := strconv.Atoi(f[0])
+		if err != nil {
+			continue
+		}
+		g := GPUSensor{Index: idx, TempC: num(f[1]), PowerW: num(f[2]), PowerLimitW: num(f[3])}
+		if bits, err := strconv.ParseUint(strings.TrimPrefix(f[4], "0x"), 16, 64); err == nil {
+			for _, b := range throttleBits {
+				if bits&b.bit != 0 {
+					g.Throttle = append(g.Throttle, b.name)
+				}
+			}
+		}
+		rest := f[5:]
+		if busy && len(rest) >= 6 {
+			g.UtilPercent, g.MemUtilPercent = numOrUnknown(rest[0]), numOrUnknown(rest[1])
+			g.MemoryUsedMB, g.MemoryTotalMB = num(rest[2]), num(rest[3])
+			g.SMClockMHz, g.ECCUncorrected = num(rest[4]), numOrUnknown(rest[5])
+			rest = rest[6:]
+		}
+		if len(rest) > 0 {
+			g.MemoryTempC = num(rest[0])
+		}
+		gs = append(gs, g)
+	}
+	return gs
+}
+
+// numOrUnknown is a figure, or -1 where nvidia-smi says it has none: a card
+// without ECC has no error count, which is not the same as none.
+func numOrUnknown(s string) float64 {
+	v, err := strconv.ParseFloat(s, 64)
+	if err != nil || v < 0 {
+		return -1
+	}
+	return v
+}
+
+// num reads a figure nvidia-smi may give as "[N/A]" or "N/A".
+func num(s string) float64 {
+	v, err := strconv.ParseFloat(s, 64)
+	if err != nil || v < 0 {
+		return 0
+	}
+	return v
+}
+
+func readTrim(path string) string {
+	b, err := os.ReadFile(path) //nolint:gosec // sysfs
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(b))
+}

internal/agent/discovery/sensors_test.go

@@ -0,0 +1,210 @@
+package discovery
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func put(t *testing.T, root, rel, body string) {
+	t.Helper()
+	p := filepath.Join(root, rel)
+	if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(p, []byte(body+"\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// dellHost is the layout of the Proxmox host these tests were written
+// against: Dell's SMM with a CPU and SODIMM sensors, coretemp with a package
+// and cores, and RAPL with a package and its dram subdomain.
+func dellHost(t *testing.T) string {
+	root := t.TempDir()
+	put(t, root, "class/hwmon/hwmon0/name", "dell_smm")
+	put(t, root, "class/hwmon/hwmon0/temp1_input", "55000")
+	put(t, root, "class/hwmon/hwmon0/temp1_label", "CPU")
+	put(t, root, "class/hwmon/hwmon0/temp2_input", "47000")
+	put(t, root, "class/hwmon/hwmon0/temp2_label", "SODIMM")
+	put(t, root, "class/hwmon/hwmon1/name", "coretemp")
+	put(t, root, "class/hwmon/hwmon1/temp1_input", "58000")
+	put(t, root, "class/hwmon/hwmon1/temp1_label", "Package id 0")
+	put(t, root, "class/hwmon/hwmon1/temp10_input", "61000")
+	put(t, root, "class/hwmon/hwmon1/temp10_label", "Core 8")
+	put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1000000000")
+	put(t, root, "class/powercap/intel-rapl:0/max_energy_range_uj", "262143328850")
+	put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "5000000")
+	return root
+}
+
+func sampler(sys string, smi func(context.Context, string) ([]byte, error), clock *time.Time) *SensorSampler {
+	return &SensorSampler{sys: sys, smi: smi, now: func() time.Time { return *clock }}
+}
+
+func noSMI(context.Context, string) ([]byte, error) { return nil, errors.New("not found") }
+
+func TestAHostsCPUHeatAndPowerAreReadAsLmSensorsWould(t *testing.T) {
+	root := dellHost(t)
+	now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	s := sampler(root, noSMI, &now)
+
+	first := s.Sample(t.Context())
+	// The package reading, not the hotter core: the package is what the
+	// chip itself reports as its temperature.
+	if first.CPUTempC != 58 {
+		t.Errorf("CPU %.1f °C, expected the package's 58", first.CPUTempC)
+	}
+	if first.CPUPowerW != 0 {
+		t.Error("power from one reading of a counter is invented")
+	}
+	if first.InletTempC != 0 || first.GPUs != nil {
+		t.Errorf("sensors this host does not have were reported: %+v", first)
+	}
+
+	// 10 s later the package has used 650 J, and its dram subdomain some of
+	// that — counted once.
+	put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1650000000")
+	put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "99000000")
+	now = now.Add(10 * time.Second)
+	if got := s.Sample(t.Context()).CPUPowerW; got < 64.9 || got > 65.1 {
+		t.Errorf("CPU power %.2f W, expected 65", got)
+	}
+}
+
+func TestACounterThatWrapsIsNotANegativeDraw(t *testing.T) {
+	root := dellHost(t)
+	now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	put(t, root, "class/powercap/intel-rapl:0/energy_uj", "262143000000")
+	s := sampler(root, noSMI, &now)
+	s.Sample(t.Context())
+	// 328,850 µJ to the wrap, then 499,671,150 past it: 500 J in 10 s.
+	put(t, root, "class/powercap/intel-rapl:0/energy_uj", "499671150")
+	now = now.Add(10 * time.Second)
+	if got := s.Sample(t.Context()).CPUPowerW; got < 49.9 || got > 50.1 {
+		t.Errorf("across a wrap: %.2f W, expected 50", got)
+	}
+}
+
+func TestAVMHasNoSensorsAndSaysNothing(t *testing.T) {
+	root := t.TempDir()
+	put(t, root, "class/thermal/cooling_device0/type", "Processor")
+	now := time.Now()
+	got := sampler(root, noSMI, &now).Sample(t.Context())
+	if got.CPUTempC != 0 || got.CPUPowerW != 0 || got.InletTempC != 0 || got.GPUs != nil {
+		t.Errorf("a machine with no sensors reported %+v", got)
+	}
+}
+
+func TestABoardsInletSensorIsTheRoomNotTheCPU(t *testing.T) {
+	root := t.TempDir()
+	put(t, root, "class/hwmon/hwmon2/name", "nct6798")
+	put(t, root, "class/hwmon/hwmon2/temp1_input", "24500")
+	put(t, root, "class/hwmon/hwmon2/temp1_label", "SYSTIN")
+	put(t, root, "class/hwmon/hwmon3/name", "k10temp")
+	put(t, root, "class/hwmon/hwmon3/temp1_input", "71250")
+	put(t, root, "class/hwmon/hwmon3/temp1_label", "Tctl")
+	now := time.Now()
+	got := sampler(root, noSMI, &now).Sample(t.Context())
+	if got.InletTempC != 24.5 || got.CPUTempC != 71.25 {
+		t.Errorf("inlet %.2f, CPU %.2f", got.InletTempC, got.CPUTempC)
+	}
+}
+
+func TestEachGPUSaysHowHotHowHungryAndWhatHoldsItBack(t *testing.T) {
+	var asked []string
+	smi := func(_ context.Context, fields string) ([]byte, error) {
+		asked = append(asked, fields)
+		return []byte(
+			"0, 71, 612.45, 700.00, 0x0000000000000000, 84\n" +
+				"1, 88, 698.10, 700.00, 0x0000000000000024, 95\n" + // sw_power_cap + sw_thermal
+				"2, 45, [N/A], 700.00, 0x0000000000000001, [N/A]\n"), nil // idle only
+	}
+	now := time.Now()
+	ss := sampler(t.TempDir(), smi, &now)
+	got := ss.Sample(t.Context())
+	if len(got.GPUs) != 3 {
+		t.Fatalf("%+v", got.GPUs)
+	}
+	g0, g1, g2 := got.GPUs[0], got.GPUs[1], got.GPUs[2]
+	if g0.TempC != 71 || g0.PowerW != 612.45 || g0.PowerLimitW != 700 || g0.MemoryTempC != 84 || g0.Throttle != nil {
+		t.Errorf("card 0 %+v", g0)
+	}
+	if strings.Join(g1.Throttle, ",") != "sw_power_cap,sw_thermal" {
+		t.Errorf("card 1 throttled by %v", g1.Throttle)
+	}
+	if g2.PowerW != 0 || g2.MemoryTempC != 0 || g2.Throttle != nil {
+		t.Errorf("card 2: N/A is not a reading, and idle is not being held back: %+v", g2)
+	}
+	// Within ten seconds the last reading is reused rather than running
+	// nvidia-smi on every heartbeat; after, it is asked again.
+	now = now.Add(5 * time.Second)
+	ss.Sample(t.Context())
+	if len(asked) != 1 {
+		t.Errorf("nvidia-smi was asked %d times within ten seconds", len(asked))
+	}
+	now = now.Add(6 * time.Second)
+	ss.Sample(t.Context())
+	if len(asked) != 2 {
+		t.Errorf("nvidia-smi was asked %d times after eleven seconds, expected 2", len(asked))
+	}
+}
+
+func TestADriverThatDoesNotKnowMemoryTemperatureStillReportsTheRest(t *testing.T) {
+	smi := func(_ context.Context, fields string) ([]byte, error) {
+		if strings.Contains(fields, "temperature.memory") {
+			return nil, errors.New(`Field "temperature.memory" is not a valid field to query.`)
+		}
+		return []byte("0, 66, 250.0, 300.0, 0x0000000000000040\n"), nil
+	}
+	now := time.Now()
+	got := sampler(t.TempDir(), smi, &now).Sample(t.Context())
+	if len(got.GPUs) != 1 || got.GPUs[0].TempC != 66 || strings.Join(got.GPUs[0].Throttle, ",") != "hw_thermal" {
+		t.Errorf("%+v", got.GPUs)
+	}
+}
+
+func TestEachSampleSaysHowBusyEachGPUIs(t *testing.T) {
+	// What the card is doing now, every sample: discovery read it once, at
+	// start, and a card busy since looked as idle as it was then.
+	smi := func(_ context.Context, fields string) ([]byte, error) {
+		if !strings.Contains(fields, "utilization.gpu") {
+			return nil, errors.New("asked without how busy the cards are")
+		}
+		return []byte(
+			"0, 71, 612.45, 700.00, 0x0000000000000000, 97, 64, 71234, 81559, 1980, 0, 84\n" +
+				"1, 30, 70.10, 700.00, 0x0000000000000001, 0, 0, 512, 81559, 345, [N/A], [N/A]\n"), nil
+	}
+	now := time.Now()
+	got := sampler(t.TempDir(), smi, &now).Sample(t.Context()).GPUs
+	if len(got) != 2 {
+		t.Fatalf("%+v", got)
+	}
+	g0, g1 := got[0], got[1]
+	if g0.UtilPercent != 97 || g0.MemUtilPercent != 64 || g0.MemoryUsedMB != 71234 || g0.MemoryTotalMB != 81559 ||
+		g0.SMClockMHz != 1980 || g0.ECCUncorrected != 0 || g0.MemoryTempC != 84 {
+		t.Errorf("card 0 %+v", g0)
+	}
+	// Idle is zero; a count the card does not keep is unknown, not zero.
+	if g1.UtilPercent != 0 || g1.ECCUncorrected != -1 || g1.MemoryTempC != 0 {
+		t.Errorf("card 1 %+v", g1)
+	}
+}
+
+func TestADriverThatDoesNotKnowTheBusyFieldsStillReportsHeatAndPower(t *testing.T) {
+	smi := func(_ context.Context, fields string) ([]byte, error) {
+		if strings.Contains(fields, "utilization.gpu") {
+			return nil, errors.New(`Field "ecc.errors.uncorrected.volatile.total" is not a valid field to query.`)
+		}
+		return []byte("0, 66, 250.0, 300.0, 0x0000000000000000, 70\n"), nil
+	}
+	now := time.Now()
+	got := sampler(t.TempDir(), smi, &now).Sample(t.Context()).GPUs
+	if len(got) != 1 || got[0].TempC != 66 || got[0].MemoryTempC != 70 || got[0].MemoryTotalMB != 0 {
+		t.Errorf("%+v", got)
+	}
+}

internal/agent/discovery/sminfo.go

@@ -0,0 +1,172 @@
+package discovery
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// ErrNoDiags means infiniband-diags is not installed, so the subnet manager
+// cannot be asked who it is.
+var ErrNoDiags = errors.New("sminfo not found: install infiniband-diags")
+
+// smLine is sminfo's one line of answer:
+//
+//	sminfo: sm lid 1 sm guid 0x98039b03009fcfd6, activity count 5452 priority 15 state 3 SMINFO_MASTER
+var smLine = regexp.MustCompile(`sm lid (\d+) sm guid (0x[0-9a-fA-F]+), activity count \d+ priority (\d+) state \d+ (\S+)`)
+
+// ParseSMInfo reads sminfo's output.
+func ParseSMInfo(out string) (*SMInfo, error) {
+	m := smLine.FindStringSubmatch(out)
+	if m == nil {
+		return nil, fmt.Errorf("unrecognised sminfo output: %q", strings.TrimSpace(out))
+	}
+	lid, _ := strconv.Atoi(m[1])
+	prio, _ := strconv.Atoi(m[3])
+	return &SMInfo{GUID: normalizeGUID(m[2]), LID: lid, Priority: prio, State: m[4]}, nil
+}
+
+// QuerySM asks, through one port, which subnet manager is master. It is a
+// management datagram to the SM, not a scan: cheap, but not free, which is
+// why the agent asks once a minute rather than every heartbeat.
+func QuerySM(ctx context.Context, hca string, port int) (*SMInfo, error) {
+	bin, err := exec.LookPath("sminfo")
+	if err != nil {
+		return nil, ErrNoDiags
+	}
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+	out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port)).CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("sminfo on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(out)))
+	}
+	return ParseSMInfo(string(out))
+}
+
+// AttachSM fills in the subnet manager each active InfiniBand port sees.
+// RoCE has none. The first error is returned for reporting; the ports it
+// could answer for keep their answer.
+func AttachSM(hcas []HCA, query func(hca string, port int) (*SMInfo, error)) error {
+	var first error
+	for i := range hcas {
+		for j := range hcas[i].Ports {
+			p := &hcas[i].Ports[j]
+			if !p.Active() || p.LinkLayer == LinkEthernet {
+				continue
+			}
+			sm, err := query(hcas[i].Name, p.Num)
+			if err != nil {
+				if first == nil {
+					first = err
+				}
+				continue
+			}
+			p.SM = sm
+		}
+	}
+	return first
+}
+
+// SMRecord is one subnet manager on a fabric, as the subnet administrator
+// lists them: master and standbys alike.
+type SMRecord struct {
+	GUID     string `json:"guid"`
+	LID      int    `json:"lid"`
+	Priority int    `json:"priority"`
+	// State is master, standby, discovering or not active.
+	State string `json:"state"`
+}
+
+// smStates are SMInfo's state numbers.
+var smStates = map[string]string{"0": "not active", "1": "discovering", "2": "standby", "3": "master"}
+
+// smirField is one line of saquery's record dump: "LID.........1".
+var smirField = regexp.MustCompile(`^\s*([A-Za-z_]+)\.{2,}\s*(\S+)`)
+
+// ParseSMIR reads `saquery SMIR`: every subnet manager the fabric's subnet
+// administrator knows of, one "SMInfoRecord dump:" block each.
+func ParseSMIR(out string) ([]SMRecord, error) {
+	var recs []SMRecord
+	var cur *SMRecord
+	for _, line := range strings.Split(out, "\n") {
+		if strings.Contains(line, "SMInfoRecord dump") {
+			recs = append(recs, SMRecord{})
+			cur = &recs[len(recs)-1]
+			continue
+		}
+		m := smirField.FindStringSubmatch(line)
+		if m == nil || cur == nil {
+			continue
+		}
+		switch strings.ToLower(m[1]) {
+		case "lid":
+			cur.LID, _ = strconv.Atoi(m[2])
+		case "guid":
+			cur.GUID = normalizeGUID(m[2])
+		case "priority":
+			cur.Priority, _ = strconv.Atoi(m[2])
+		case "smstate":
+			cur.State = smStates[m[2]]
+			if cur.State == "" {
+				cur.State = m[2]
+			}
+		}
+	}
+	out2 := recs[:0]
+	for _, r := range recs {
+		if r.GUID != "" {
+			out2 = append(out2, r)
+		}
+	}
+	if len(out2) == 0 {
+		return nil, fmt.Errorf("no subnet manager records in saquery output")
+	}
+	return out2, nil
+}
+
+// QuerySMs asks the subnet administrator, through one port, for every subnet
+// manager on the fabric.
+func QuerySMs(ctx context.Context, hca string, port int) ([]SMRecord, error) {
+	bin, err := exec.LookPath("saquery")
+	if err != nil {
+		return nil, ErrNoDiags
+	}
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+	out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port), "SMIR").CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("saquery on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(out)))
+	}
+	return ParseSMIR(string(out))
+}
+
+// ScanFabric maps the fabric a port is on with ibnetdiscover: every switch,
+// adapter and link. It can take a while on a large fabric, so it is bounded,
+// and its output is capped: a fabric of thousands of ports is a few
+// megabytes of text.
+func ScanFabric(ctx context.Context, hca string, port int) (string, error) {
+	bin, err := exec.LookPath("ibnetdiscover")
+	if err != nil {
+		return "", ErrNoDiags
+	}
+	ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
+	defer cancel()
+	out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port)).Output()
+	if err != nil {
+		var ee *exec.ExitError
+		if errors.As(err, &ee) {
+			return "", fmt.Errorf("ibnetdiscover on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(ee.Stderr)))
+		}
+		return "", fmt.Errorf("ibnetdiscover on %s/%d: %w", hca, port, err)
+	}
+	const limit = 16 << 20
+	if len(out) > limit {
+		return "", fmt.Errorf("ibnetdiscover printed %d MB, more than the %d MB a scan may send", len(out)>>20, limit>>20)
+	}
+	return string(out), nil
+}

internal/agent/discovery/topo.go

@@ -0,0 +1,185 @@
+package discovery
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// Topology is what `nvidia-smi topo -m` says about how a node's GPUs and
+// network adapters are wired together.
+//
+// Read from nvidia-smi rather than NVML so the portable, cgo-free binary knows
+// it too. Without it that binary saw no NVLink at all: every GPU reported no
+// peers, and a tensor-parallel group was chosen by index on machines where
+// NVLink decides whether it runs at full speed.
+type Topology struct {
+	// NVLink is, per GPU, the GPUs it reaches over NVLink and how many links
+	// are bonded to each.
+	NVLink map[int]map[int]int
+	// NICPath is, per GPU, how far each network adapter is, by its HCA name:
+	// PIX, PXB, PHB, NODE or SYS, nearest first. A GPU should talk to the
+	// fabric through the adapter nearest it; one across the socket costs a
+	// trip over the processor interconnect on every transfer.
+	NICPath map[int]map[string]string
+	// NUMA is the NUMA node each GPU is attached to, where reported.
+	NUMA map[int]int
+}
+
+// PathRank orders the relations nvidia-smi reports, nearest first. Lower is
+// closer; an unknown relation ranks last.
+func PathRank(rel string) int {
+	switch strings.ToUpper(rel) {
+	case "PIX":
+		return 0
+	case "PXB":
+		return 1
+	case "PHB":
+		return 2
+	case "NODE":
+		return 3
+	case "SYS":
+		return 4
+	}
+	return 5
+}
+
+// DiscoverTopology runs `nvidia-smi topo -m`. A node without nvidia-smi has
+// no topology to report, which is not an error.
+func DiscoverTopology(ctx context.Context, binary string) (Topology, error) {
+	if binary == "" {
+		if _, err := exec.LookPath("nvidia-smi"); err != nil {
+			return Topology{}, nil
+		}
+		binary = "nvidia-smi"
+	}
+	out, err := exec.CommandContext(ctx, binary, "topo", "-m").Output()
+	if err != nil {
+		return Topology{}, fmt.Errorf("running %s topo -m: %w", binary, err)
+	}
+	return parseTopo(string(out))
+}
+
+// ParseTopology reads `nvidia-smi topo -m` output.
+func ParseTopology(out string) (Topology, error) { return parseTopo(out) }
+
+var (
+	ansi       = regexp.MustCompile(`\x1b\[[0-9;]*m`)
+	nvLinks    = regexp.MustCompile(`^NV(\d+)$`)
+	nicLegend  = regexp.MustCompile(`^\s*(NIC\d+)\s*:\s*(\S+)\s*$`)
+	gpuColumn  = regexp.MustCompile(`^GPU(\d+)$`)
+	numaColumn = "NUMA Affinity"
+)
+
+// parseTopo reads the matrix. Columns are tab-separated; the header names
+// each one — GPUn, then the adapters, as NICn with a legend below on newer
+// drivers or by HCA name on older ones — then CPU and NUMA affinity.
+func parseTopo(out string) (Topology, error) {
+	out = ansi.ReplaceAllString(out, "")
+	lines := strings.Split(out, "\n")
+
+	legend := map[string]string{}
+	for _, l := range lines {
+		if m := nicLegend.FindStringSubmatch(l); m != nil {
+			legend[m[1]] = m[2]
+		}
+	}
+
+	var header []string
+	start := -1
+	for i, l := range lines {
+		cells := splitTabs(l)
+		if len(cells) > 1 && cells[0] == "" && gpuColumn.MatchString(cells[1]) {
+			header, start = cells, i
+			break
+		}
+	}
+	if start < 0 {
+		return Topology{}, fmt.Errorf("no GPU matrix in nvidia-smi topo output")
+	}
+
+	t := Topology{NVLink: map[int]map[int]int{}, NICPath: map[int]map[string]string{}, NUMA: map[int]int{}}
+	for _, l := range lines[start+1:] {
+		cells := splitTabs(l)
+		if len(cells) < 2 {
+			// The matrix ends at the first blank line, before the legend.
+			if strings.TrimSpace(l) == "" {
+				break
+			}
+			continue
+		}
+		m := gpuColumn.FindStringSubmatch(cells[0])
+		if m == nil {
+			continue // an adapter's row: its view of the GPUs is the same matrix
+		}
+		gpu, _ := strconv.Atoi(m[1])
+		for c := 1; c < len(cells) && c < len(header); c++ {
+			col, v := header[c], strings.TrimSpace(cells[c])
+			switch {
+			case gpuColumn.MatchString(col):
+				peer, _ := strconv.Atoi(gpuColumn.FindStringSubmatch(col)[1])
+				if n := nvLinks.FindStringSubmatch(v); n != nil && peer != gpu {
+					links, _ := strconv.Atoi(n[1])
+					if t.NVLink[gpu] == nil {
+						t.NVLink[gpu] = map[int]int{}
+					}
+					t.NVLink[gpu][peer] = links
+				}
+			case col == numaColumn:
+				if n, err := strconv.Atoi(v); err == nil {
+					t.NUMA[gpu] = n
+				}
+			case col == "CPU Affinity" || col == "GPU NUMA ID" || col == "":
+			default:
+				// An adapter: NICn named by the legend, or an HCA by name.
+				hca := col
+				if name, ok := legend[col]; ok {
+					hca = name
+				}
+				if PathRank(v) < 5 {
+					if t.NICPath[gpu] == nil {
+						t.NICPath[gpu] = map[string]string{}
+					}
+					t.NICPath[gpu][hca] = strings.ToUpper(v)
+				}
+			}
+		}
+	}
+	if len(t.NVLink) == 0 && len(t.NICPath) == 0 && len(t.NUMA) == 0 {
+		return t, fmt.Errorf("the nvidia-smi topo matrix had no rows")
+	}
+	return t, nil
+}
+
+func splitTabs(l string) []string {
+	l = strings.TrimRight(l, "\r")
+	cells := strings.Split(l, "\t")
+	for i := range cells {
+		cells[i] = strings.TrimSpace(cells[i])
+	}
+	return cells
+}
+
+// Apply puts the topology on the GPUs it describes. What a GPU already knows
+// — NVLink peers from NVML, which is the more exact source — is kept.
+func (t Topology) Apply(gpus []GPU) {
+	for i := range gpus {
+		g := &gpus[i]
+		if len(g.NVLinkPeers) == 0 {
+			for peer := range t.NVLink[g.Index] {
+				g.NVLinkPeers = append(g.NVLinkPeers, peer)
+			}
+			sort.Ints(g.NVLinkPeers)
+		}
+		if n, ok := t.NUMA[g.Index]; ok {
+			g.NUMANode = n
+		}
+		if paths := t.NICPath[g.Index]; len(paths) > 0 {
+			g.NICPaths = paths
+		}
+	}
+}

internal/agent/discovery/topo_test.go

@@ -0,0 +1,113 @@
+package discovery
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// row joins cells the way nvidia-smi prints them: tab-separated, each padded.
+func row(cells ...string) string { return strings.Join(cells, "\t") }
+
+// dgxH100 is `nvidia-smi topo -m` on an eight-GPU NVSwitch machine with two
+// adapters, as newer drivers print it: NICn columns and a legend naming them.
+var dgxH100 = strings.Join([]string{
+	row("\x1b[4m", "GPU0", "GPU1", "GPU2", "GPU3", "NIC0", "NIC1", "CPU Affinity", "NUMA Affinity", "GPU NUMA ID\x1b[0m"),
+	row("GPU0", " X ", "NV18", "NV18", "NV18", "PXB", "SYS", "0-55,112-167", "0", "N/A"),
+	row("GPU1", "NV18", " X ", "NV18", "NV18", "PXB", "SYS", "0-55,112-167", "0", "N/A"),
+	row("GPU2", "NV18", "NV18", " X ", "NV18", "SYS", "PXB", "56-111,168-223", "1", "N/A"),
+	row("GPU3", "NV18", "NV18", "NV18", " X ", "SYS", "PIX", "56-111,168-223", "1", "N/A"),
+	row("NIC0", "PXB", "PXB", "SYS", "SYS", " X ", "SYS"),
+	row("NIC1", "SYS", "SYS", "PXB", "PIX", "SYS", " X "),
+	"",
+	"Legend:",
+	"",
+	"  X    = Self",
+	"  SYS  = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)",
+	"  NV#  = Connection traversing a bonded set of # NVLinks",
+	"",
+	"NIC Legend:",
+	"",
+	"  NIC0: mlx5_0",
+	"  NIC1: mlx5_1",
+	"",
+}, "\n")
+
+func TestNVLinkAndAdapterDistanceAreReadFromTopo(t *testing.T) {
+	topo, err := parseTopo(dgxH100)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := topo.NVLink[0]; !reflect.DeepEqual(got, map[int]int{1: 18, 2: 18, 3: 18}) {
+		t.Errorf("GPU0's NVLink peers = %v", got)
+	}
+	// The legend turns NIC1 into the adapter's own name.
+	if got := topo.NICPath[3]; !reflect.DeepEqual(got, map[string]string{"mlx5_0": "SYS", "mlx5_1": "PIX"}) {
+		t.Errorf("GPU3's adapters = %v", got)
+	}
+	if topo.NUMA[2] != 1 || topo.NUMA[0] != 0 {
+		t.Errorf("NUMA = %v", topo.NUMA)
+	}
+
+	gpus := []GPU{{Index: 0}, {Index: 3}}
+	topo.Apply(gpus)
+	if !reflect.DeepEqual(gpus[0].NVLinkPeers, []int{1, 2, 3}) || gpus[1].NUMANode != 1 ||
+		gpus[1].NICPaths["mlx5_1"] != "PIX" {
+		t.Errorf("applied = %+v", gpus)
+	}
+}
+
+func TestOlderDriversNameAdaptersInTheHeader(t *testing.T) {
+	// No legend: the columns are the adapters' own names, and NVLink between
+	// pairs only, as on an A100 PCIe box with bridges.
+	out := strings.Join([]string{
+		row("", "GPU0", "GPU1", "GPU2", "GPU3", "mlx5_0", "CPU Affinity", "NUMA Affinity"),
+		row("GPU0", " X ", "NV12", "SYS", "SYS", "NODE", "0-31", "0"),
+		row("GPU1", "NV12", " X ", "SYS", "SYS", "NODE", "0-31", "0"),
+		row("GPU2", "SYS", "SYS", " X ", "NV12", "SYS", "32-63", "1"),
+		row("GPU3", "SYS", "SYS", "NV12", " X ", "SYS", "32-63", "1"),
+		row("mlx5_0", "NODE", "NODE", "SYS", "SYS", " X "),
+		"",
+	}, "\n")
+	topo, err := parseTopo(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(topo.NVLink[2], map[int]int{3: 12}) || len(topo.NVLink[0]) != 1 {
+		t.Errorf("NVLink = %v", topo.NVLink)
+	}
+	if topo.NICPath[0]["mlx5_0"] != "NODE" {
+		t.Errorf("adapters = %v", topo.NICPath)
+	}
+}
+
+func TestAPCIeMachineHasNoNVLink(t *testing.T) {
+	out := strings.Join([]string{
+		row("", "GPU0", "GPU1", "CPU Affinity", "NUMA Affinity"),
+		row("GPU0", " X ", "PHB", "0-15", "0"),
+		row("GPU1", "PHB", " X ", "0-15", "0"),
+		"",
+	}, "\n")
+	topo, err := parseTopo(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(topo.NVLink) != 0 {
+		t.Errorf("NVLink on a PCIe machine: %v", topo.NVLink)
+	}
+}
+
+func TestNVMLPeersAreKeptOverTopo(t *testing.T) {
+	topo, _ := parseTopo(dgxH100)
+	gpus := []GPU{{Index: 0, NVLinkPeers: []int{1}}}
+	topo.Apply(gpus)
+	if !reflect.DeepEqual(gpus[0].NVLinkPeers, []int{1}) {
+		t.Errorf("NVML's answer was replaced: %v", gpus[0].NVLinkPeers)
+	}
+}
+
+func TestSomethingThatIsNotAMatrixIsAnError(t *testing.T) {
+	if _, err := parseTopo("NVIDIA-SMI has failed because it couldn't communicate with the NVIDIA driver."); err == nil {
+		t.Error("expected an error")
+	}
+}

internal/agent/engines/engine.go

@@ -9,6 +9,7 @@
 package engines
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"slices"
@@ -40,6 +41,9 @@ type Profile struct {
 	ModelPath string
 	// ServedModelName is the name clients use, which need not match the path.
 	ServedModelName string
+	// Adapters are LoRA adapters to load beside the model, in order: an
+	// adapter's index is the id requests apply it by.
+	Adapters []Adapter
 }
 
 // ShardSpec is one shard of one replica on this node.
@@ -63,13 +67,28 @@ type ShardSpec struct {
 	HCAs []string
 	// MgmtInterface is the ethernet interface used for bootstrap traffic.
 	MgmtInterface string
+	// GIDIndex is the RoCE v2 GID index for the HCAs, set by the agent when
+	// they are RoCE; nil on InfiniBand, where NCCL needs none.
+	GIDIndex *int
 
 	HTTPPort int
 	Profile  Profile
 
+	// APIKey is what the engine demands of every request. Engines listen on
+	// all of a node's addresses; without a key anybody who can reach the port
+	// talks to the model directly, past the gateway's authentication, quotas
+	// and accounting. Given to the engine in its environment rather than on
+	// its command line, which every user on the node can read.
+	APIKey string `json:"api_key,omitempty"`
+	// Threads is the CPU cores placement reserved for a shard with no GPU,
+	// and so what its engine computes on. Zero leaves it to the engine's own
+	// configuration.
+	Threads int `json:"threads,omitempty"`
+
 	// Pool and Fabric are control plane concepts the agent cannot derive. They
-	// are carried so the agent can report them back after a restart, and are
-	// not used when building a command.
+	// are carried so the agent can report them back after a restart; Fabric
+	// also says, with an "nvl:" prefix, that the replica is inside a
+	// multi-node NVLink domain, which changes NCCL's environment.
 	Pool   string
 	Fabric string
 	// ProfileFP identifies the version of the profile this shard was started
@@ -128,6 +147,15 @@ type Engine interface {
 	MetricsPath() string
 }
 
+// CPUFallback is an engine that can come up on the CPU when it was placed on
+// GPUs, and keep serving there without an error. The agent checks such a shard
+// is on its GPUs before calling it ready.
+type CPUFallback interface {
+	FallsBackToCPU() bool
+	// Devices lists the GPUs the engine's build can see, for saying why.
+	Devices(ctx context.Context) ([]string, error)
+}
+
 // GPUInfo is the subset of GPU facts validation needs.
 type GPUInfo struct {
 	Index             int
@@ -185,6 +213,10 @@ func (r *Registry) Names() []string {
 
 // Config configures the engine adapters.
 type Config struct {
+	// Disabled are engines this node never installs or advertises, whatever
+	// the control plane asks: vLLM on a node of cards no build of it runs on,
+	// that is meant for llama.cpp alone.
+	Disabled []string       `koanf:"disabled"`
 	VLLM     VLLMConfig     `koanf:"vllm"`
 	SGLang   SGLangConfig   `koanf:"sglang"`
 	LlamaCpp LlamaCppConfig `koanf:"llamacpp"`
@@ -193,6 +225,10 @@ type Config struct {
 
 // --- shared helpers ---
 
+// NCCLEnv is the environment a shard with this spec is given, for a tool that
+// shows an operator what an engine would see.
+func NCCLEnv(s ShardSpec) map[string]string { return baseEnv(s) }
+
 // baseEnv builds the environment every engine shares.
 //
 // This is the part of §5.3 the plan marks critical. The rules encoded here:
@@ -210,16 +246,34 @@ type Config struct {
 func baseEnv(s ShardSpec) map[string]string {
 	env := map[string]string{
 		"CUDA_VISIBLE_DEVICES": joinInts(s.GPUIndices),
+		// The indices above are nvidia-smi's, which counts by PCI bus. CUDA
+		// counts fastest first unless told otherwise, and on a node with two
+		// kinds of GPU the shard would get a different card than placement
+		// gave it — one another shard may hold.
+		"CUDA_DEVICE_ORDER": "PCI_BUS_ID",
 	}
 
 	if len(s.HCAs) > 0 {
 		env["NCCL_IB_HCA"] = strings.Join(s.HCAs, ",")
 		env["NCCL_IB_DISABLE"] = "0"
+		// RoCE: the index of the routable RoCE v2 entry. NCCL's default is a
+		// v1 link-local one on most hosts, and a job on it hangs.
+		if s.GIDIndex != nil {
+			env["NCCL_IB_GID_INDEX"] = strconv.Itoa(*s.GIDIndex)
+		}
 	} else {
 		// Single node, or a node with no fabric. Either way IB is not the path.
 		env["NCCL_IB_DISABLE"] = "1"
 	}
 
+	// Placed inside a multi-node NVLink domain: NCCL is to carry the
+	// replica's collectives over NVLink between its nodes, which it only
+	// does when told to, and needs IMEX for.
+	if s.NNodes > 1 && strings.HasPrefix(s.Fabric, "nvl:") {
+		env["NCCL_MNNVL_ENABLE"] = "1"
+		env["NCCL_CUMEM_ENABLE"] = "1"
+	}
+
 	if s.MgmtInterface != "" {
 		env["NCCL_SOCKET_IFNAME"] = s.MgmtInterface
 		env["GLOO_SOCKET_IFNAME"] = s.MgmtInterface
@@ -228,6 +282,11 @@ func baseEnv(s ShardSpec) map[string]string {
 	// Profile overrides win: they are the escape hatch for the per-cluster
 	// tuning no adapter can guess.
 	for k, v := range s.Profile.Env {
+		// Checked again here, not only where the model was defined: this is
+		// the process about to run as root, whatever sent the profile.
+		if bad, _ := UnsafeEnv(k); bad {
+			continue
+		}
 		env[k] = v
 	}
 	return env
@@ -336,3 +395,10 @@ func quantizationOf(args []string) string {
 	}
 	return ""
 }
+
+// Adapter is a LoRA adapter an engine loads beside its model. Path is a
+// source until the agent has fetched it, a local file after.
+type Adapter struct {
+	Name string `json:"name"`
+	Path string `json:"path"`
+}

internal/agent/engines/engines_test.go

@@ -3,6 +3,7 @@ package engines
 import (
 	"fmt"
 	"slices"
+	"strconv"
 	"strings"
 	"testing"
 )
@@ -243,14 +244,41 @@ func TestVLLMRayWorkerRunsOnlyRay(t *testing.T) {
 }
 
 func TestVLLMMPMode(t *testing.T) {
+	// vLLM's multi-node launch: every node the same serve with its rank and
+	// the leader's address, workers headless. Workers used to be given the
+	// leader's address as their own VLLM_HOST_IP, and no rank at all.
 	v := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}
-	cmd := buildOne(t, v, multiNodeSpec(0))
-	if argValue(cmd.Args, "--distributed-executor-backend") != "mp" {
-		t.Errorf("backend not set: %v", cmd.Args)
+	for _, rank := range []int{0, 1} {
+		cmd := buildOne(t, v, multiNodeSpec(rank))
+		for flag, want := range map[string]string{
+			"--distributed-executor-backend": "mp", "--nnodes": "2",
+			"--node-rank": strconv.Itoa(rank), "--master-addr": "10.0.0.10", "--master-port": "20001",
+		} {
+			if got := argValue(cmd.Args, flag); got != want {
+				t.Errorf("rank %d: %s = %q, want %q (%v)", rank, flag, got, want, cmd.Args)
+			}
+		}
+		if headless := slices.Contains(cmd.Args, "--headless"); headless != (rank > 0) {
+			t.Errorf("rank %d: headless = %v", rank, headless)
+		}
+		if ip, ok := envMap(cmd.Env)["VLLM_HOST_IP"]; ok && ip == "10.0.0.10" && rank > 0 {
+			t.Errorf("a worker was told the leader's address is its own")
+		}
 	}
-	env := envMap(cmd.Env)
-	if env["VLLM_HOST_IP"] != "10.0.0.10" || env["VLLM_PORT"] != "20001" {
-		t.Errorf("dist init not configured: %q %q", env["VLLM_HOST_IP"], env["VLLM_PORT"])
+}
+
+func TestRoCEGetsItsGIDIndex(t *testing.T) {
+	s := multiNodeSpec(0)
+	idx := 3
+	s.GIDIndex = &idx
+	env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, s).Env)
+	if env["NCCL_IB_GID_INDEX"] != "3" {
+		t.Errorf("NCCL_IB_GID_INDEX = %q", env["NCCL_IB_GID_INDEX"])
+	}
+	// InfiniBand needs none, and must not be given one.
+	env = envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, multiNodeSpec(0)).Env)
+	if _, ok := env["NCCL_IB_GID_INDEX"]; ok {
+		t.Error("an InfiniBand shard was given a GID index")
 	}
 }
 
@@ -423,3 +451,28 @@ func TestQuantizationOf(t *testing.T) {
 		t.Errorf("a dangling flag should yield nothing, got %q", got)
 	}
 }
+
+func TestAReplicaInAnNVLinkDomainIsToldToUseIt(t *testing.T) {
+	s := multiNodeSpec(0)
+	s.Fabric = "nvl:76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa/7"
+	env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, s).Env)
+	if env["NCCL_MNNVL_ENABLE"] != "1" {
+		t.Errorf("NCCL_MNNVL_ENABLE = %q", env["NCCL_MNNVL_ENABLE"])
+	}
+	if env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, multiNodeSpec(0)).Env); env["NCCL_MNNVL_ENABLE"] != "" {
+		t.Error("a replica on InfiniBand was told to use multi-node NVLink")
+	}
+}
+
+func TestMultiNodeVLLMDefaultsToItsOwnLaunch(t *testing.T) {
+	// mp is what vLLM documents and needs nothing but vLLM; Ray was the
+	// default and was never checked against a real cluster.
+	cmds, err := (&VLLM{}).Commands(multiNodeSpec(1))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(cmds) != 1 || cmds[0].Name != "vllm" || argValue(cmds[0].Args, "--distributed-executor-backend") != "mp" ||
+		!slices.Contains(cmds[0].Args, "--headless") {
+		t.Errorf("default worker = %+v", cmds)
+	}
+}

internal/agent/engines/llamacpp.go

@@ -1,9 +1,12 @@
 package engines
 
 import (
+	"context"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -21,8 +24,21 @@ type LlamaCppConfig struct {
 	// does not say. llama.cpp defaults to one, which would make every second
 	// request wait behind the first no matter what the router decided.
 	Parallel int `koanf:"parallel"`
+	// Threads is how many CPU threads a shard with no GPU computes on. Zero
+	// takes half this machine's CPUs.
+	//
+	// Not all of them, which is llama.cpp's own default: two busy servers
+	// that each spin on every CPU fight for them, and on an 8-vCPU node
+	// that made 7B generation fall from 10 tokens a second to under 1 each.
+	// At half, one server alone is about a third slower and two together
+	// keep 5.5 each. A node that only ever runs one CPU replica does better
+	// with all of them; say so here.
+	Threads int `koanf:"threads"`
 }
 
+// numCPU is the CPUs this process may run on; a variable for tests.
+var numCPU = runtime.NumCPU
+
 // LlamaCpp builds llama.cpp launch commands.
 //
 // It is the engine that makes a cluster without GPUs useful: llama.cpp runs a
@@ -51,25 +67,25 @@ func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true }
 
 // Validate checks the profile against the GPUs it would run on.
 //
-// The two refusals here are about not pretending: llama.cpp has no equivalent
-// of a tensor-parallel group spanning machines, and its multi-GPU support is a
-// different mechanism (--split-mode, --tensor-split) that this adapter does not
-// drive. Accepting either would produce a replica that starts and then serves
-// from one machine while the catalog believes it has several.
+// llama.cpp spreads a model over a node's GPUs by layer (--split-mode layer),
+// so gpus_per_node is how many it gets. It has no tensor parallelism and no
+// group spanning machines, and accepting tp or nodes would produce a replica
+// that serves from fewer cards, or one machine, while the catalog believes
+// otherwise.
 func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error {
-	if err := validateCommon(p, gpus); err != nil {
-		return err
-	}
 	if p.Nodes > 1 {
 		return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica",
 			p.Name, p.Nodes)
 	}
-	if p.TP > 1 {
-		return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+
-			"it splits a model across local GPUs with --tensor-split, which this adapter does not drive",
-			p.Name)
+	if p.TP > 1 || p.PP > 1 {
+		return fmt.Errorf("profile %q: llamacpp splits a model across a node's GPUs by layer, not by tp or pp; "+
+			"write gpus_per_node: %d instead", p.Name, max(p.TP, 1)*max(p.PP, 1))
 	}
-	return nil
+	// The common checks count a replica's GPUs as tp; for llama.cpp the
+	// count is gpus_per_node.
+	q := p
+	q.TP = max(p.GPUsPerNode, 1)
+	return validateCommon(q, gpus)
 }
 
 // Commands returns the processes for one shard, which is always one process.
@@ -118,10 +134,50 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
 	if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") {
 		args = append(args, "--n-gpu-layers", "999")
 	}
+	// Several GPUs: the layers spread across all of them, which is the only
+	// split llama.cpp does without a matched interconnect. In proportion to
+	// each card's free memory, which for the whole cards placement gives is
+	// an even split.
+	if len(s.GPUIndices) > 1 && !hasFlag(p.Args, "--split-mode", "-sm") {
+		args = append(args, "--split-mode", "layer")
+	}
+	if len(s.GPUIndices) == 0 && !hasFlag(p.Args, "--threads", "-t") {
+		// What placement reserved first: it is what the node's other
+		// replicas were placed around.
+		n := s.Threads
+		if n <= 0 {
+			n = l.cfg.Threads
+		}
+		if n <= 0 {
+			n = max(numCPU()/2, 1)
+		}
+		args = append(args, "--threads", strconv.Itoa(n))
+	}
+	// Loaded at start, applied per request: a request that names an adapter
+	// carries "lora": [{"id": i, "scale": s}], and one that names none is the
+	// base model. Each is loaded at scale zero, not with --lora and
+	// --lora-init-without-apply: that leaves an adapter's scale at 1, and
+	// llama-server applies it at that scale to every request with no "lora"
+	// of its own — every request for the model itself.
+	for _, a := range p.Adapters {
+		args = append(args, "--lora-scaled", a.Path+":0")
+	}
 	args = append(args, p.Args...)
 
+	env := baseEnv(s)
+	if s.APIKey != "" {
+		env["LLAMA_API_KEY"] = s.APIKey
+	}
+	// The Vulkan build — what a GPU older than the CUDA builds gets — does
+	// not read CUDA_VISIBLE_DEVICES; this is its own. Vulkan numbers devices
+	// as the driver lists them, which on NVIDIA follows the same bus order
+	// as nvidia-smi; were it not to, the shard would not hold memory on its
+	// own GPUs, and starting it checks exactly that.
+	if len(s.GPUIndices) > 0 {
+		env["GGML_VK_VISIBLE_DEVICES"] = joinInts(s.GPUIndices)
+	}
 	return []Command{{
-		Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
+		Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(env),
 	}}, nil
 }
 
@@ -195,3 +251,38 @@ func (l *LlamaCpp) bin() string {
 	}
 	return "llama-server"
 }
+
+// FallsBackToCPU says that llama.cpp keeps running when its GPU backend does
+// not load: --n-gpu-layers asks for layers on a GPU it then has none of, and
+// the server comes up and answers on the CPU, at a tenth of the speed, with no
+// error anywhere. Its readiness on a GPU placement needs checking separately.
+func (*LlamaCpp) FallsBackToCPU() bool { return true }
+
+// Devices lists the GPUs this build can compute on, as llama-server's
+// --list-devices names them ("CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)").
+// Empty, on a machine with GPUs, is a build whose GPU backend did not load:
+// the CPU build, or a CUDA build missing a library its backend links.
+func (l *LlamaCpp) Devices(ctx context.Context) ([]string, error) {
+	out, err := exec.CommandContext(ctx, l.bin(), "--list-devices").CombinedOutput() //nolint:gosec // the engine's own binary
+	if err != nil {
+		return nil, fmt.Errorf("%s --list-devices: %v: %s", l.bin(), err, strings.TrimSpace(string(out)))
+	}
+	return parseDevices(string(out)), nil
+}
+
+// parseDevices reads --list-devices: the lines after "Available devices:",
+// which says "(none)" when there are none. Log lines before it are skipped.
+func parseDevices(out string) []string {
+	var devs []string
+	listing := false
+	for _, line := range strings.Split(out, "\n") {
+		line = strings.TrimSpace(line)
+		switch {
+		case strings.HasPrefix(line, "Available devices"):
+			listing = true
+		case listing && line != "" && line != "(none)":
+			devs = append(devs, line)
+		}
+	}
+	return devs
+}

internal/agent/engines/llamacpp_test.go

@@ -245,3 +245,119 @@ func TestAMissingPathIsLeftToTheEngine(t *testing.T) {
 		t.Errorf("expected it passed through, got %q %v", got, err)
 	}
 }
+
+func TestLlamaCppIsGivenItsKeyOutOfSightOfTheProcessList(t *testing.T) {
+	// The command line is readable by every user on the node; the
+	// environment of a root process is not.
+	cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, APIKey: "k1",
+		Profile: Profile{ModelPath: "/m.gguf"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if slices.Contains(cmds[0].Args, "k1") || slices.Contains(cmds[0].Args, "--api-key") {
+		t.Errorf("the key is on the command line: %v", cmds[0].Args)
+	}
+	if !slices.Contains(cmds[0].Env, "LLAMA_API_KEY=k1") {
+		t.Error("llama-server was not given its key")
+	}
+}
+
+func TestACPUShardLeavesHalfTheMachineForAnother(t *testing.T) {
+	prev := numCPU
+	numCPU = func() int { return 8 }
+	t.Cleanup(func() { numCPU = prev })
+
+	threads := func(l *LlamaCpp, s ShardSpec) (string, bool) {
+		cmds, err := l.Commands(s)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return flagValue(cmds[0].Args, "--threads")
+	}
+	if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "4" {
+		t.Errorf("--threads %q on 8 CPUs, expected 4", v)
+	}
+	if v, _ := threads(&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}, llamaSpec(nil)); v != "8" {
+		t.Errorf("--threads %q with 8 configured", v)
+	}
+	if _, ok := threads(&LlamaCpp{}, llamaSpec([]int{0})); ok {
+		t.Error("a GPU shard was given a thread count")
+	}
+	s := llamaSpec(nil)
+	s.Profile.Args = []string{"-t", "2"}
+	cmds, _ := (&LlamaCpp{}).Commands(s)
+	if slices.Contains(cmds[0].Args, "--threads") {
+		t.Error("the profile's own -t was overridden")
+	}
+	numCPU = func() int { return 1 }
+	if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "1" {
+		t.Errorf("--threads %q on one CPU", v)
+	}
+}
+
+func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) {
+	s := llamaSpec(nil)
+	s.Threads = 3
+	cmds, err := (&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}).Commands(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v, _ := flagValue(cmds[0].Args, "--threads"); v != "3" {
+		t.Errorf("--threads %q; what placement reserved should win over the node's setting", v)
+	}
+}
+
+func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) {
+	s := llamaSpec(nil)
+	s.Profile.Adapters = []Adapter{{Name: "sql", Path: "/w/sql.gguf"}, {Name: "docs", Path: "/w/docs.gguf"}}
+	cmds, err := (&LlamaCpp{}).Commands(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	args := strings.Join(cmds[0].Args, " ")
+	if !strings.Contains(args, "--lora-scaled /w/sql.gguf:0 --lora-scaled /w/docs.gguf:0") {
+		t.Errorf("args %s: both adapters, in order, loaded at scale zero", args)
+	}
+	plain, _ := (&LlamaCpp{}).Commands(llamaSpec(nil))
+	if strings.Contains(strings.Join(plain[0].Args, " "), "--lora") {
+		t.Error("a model with no adapters was given lora flags")
+	}
+}
+
+func TestDevicesAreReadFromListDevices(t *testing.T) {
+	cuda := "ggml_cuda_init: found 2 CUDA devices:\nAvailable devices:\n  CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n  CUDA1: NVIDIA L4 (22994 MiB, 22713 MiB free)\n"
+	if got := parseDevices(cuda); len(got) != 2 || !strings.HasPrefix(got[0], "CUDA0: NVIDIA L4") {
+		t.Errorf("CUDA build: %q", got)
+	}
+	// A CUDA build whose backend did not load says what the CPU build says.
+	if got := parseDevices("Available devices:\n  (none)\n"); len(got) != 0 {
+		t.Errorf("no devices: %q", got)
+	}
+}
+
+func TestLlamaCppSpreadsAReplicaOverItsNodesGPUsByLayer(t *testing.T) {
+	four := Profile{Name: "t4x4", Engine: "llamacpp", GPUClass: "t4", TP: 1, GPUsPerNode: 4, ModelPath: "/m"}
+	if err := (&LlamaCpp{}).Validate(four, h100(4)); err != nil {
+		t.Fatalf("four GPUs on a four-GPU node: %v", err)
+	}
+	if err := (&LlamaCpp{}).Validate(four, h100(2)); err == nil {
+		t.Error("four GPUs were accepted on a node with two")
+	}
+
+	s := llamaSpec([]int{0, 1, 2, 3})
+	s.Profile.GPUsPerNode = 4
+	cmds, err := (&LlamaCpp{}).Commands(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v, _ := flagValue(cmds[0].Args, "--split-mode"); v != "layer" {
+		t.Errorf("--split-mode %q over four GPUs", v)
+	}
+	if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=0,1,2,3") {
+		t.Errorf("env %v: the replica must see all four of its GPUs", cmds[0].Env)
+	}
+	one, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
+	if hasFlag(one[0].Args, "--split-mode") {
+		t.Error("a one-GPU replica was given a split mode")
+	}
+}

internal/agent/engines/probe.go

@@ -202,11 +202,7 @@ func (g *SGLang) Configured() bool { return g.cfg.Python != "" }
 
 // Probe reports whether vLLM is installed.
 func (v *VLLM) Probe(context.Context) (string, error) {
-	bin := v.cfg.Bin
-	if bin == "" {
-		bin = "vllm"
-	}
-	return installedBinary(bin)
+	return installedBinary(v.bin())
 }
 
 // Probe reports the SGLang version.
@@ -215,10 +211,6 @@ func (v *VLLM) Probe(context.Context) (string, error) {
 // asking the shell for "sglang" finds nothing on a machine that can run it
 // perfectly well.
 func (g *SGLang) Probe(ctx context.Context) (string, error) {
-	python := g.cfg.Python
-	if python == "" {
-		python = "python3"
-	}
-	return versionOfModule(ctx, python, "-c",
+	return versionOfModule(ctx, g.python(), "-c",
 		"import sglang, sys; sys.stdout.write(getattr(sglang, '__version__', ''))")
 }

internal/agent/engines/sglang.go

@@ -1,6 +1,9 @@
 package engines
 
-import "strconv"
+import (
+	"strconv"
+	"sync/atomic"
+)
 
 // SGLangConfig locates the SGLang installation.
 type SGLangConfig struct {
@@ -11,7 +14,25 @@ type SGLangConfig struct {
 }
 
 // SGLang builds SGLang launch commands.
-type SGLang struct{ cfg SGLangConfig }
+type SGLang struct {
+	cfg       SGLangConfig
+	installed atomic.Pointer[string]
+}
+
+// UseInstalled points the engine at the interpreter of the environment the
+// agent installed and checked, ahead of the configuration.
+func (g *SGLang) UseInstalled(path string) { g.installed.Store(&path) }
+
+// python is the interpreter to launch SGLang with.
+func (g *SGLang) python() string {
+	if p := g.installed.Load(); p != nil {
+		return *p
+	}
+	if g.cfg.Python != "" {
+		return g.cfg.Python
+	}
+	return "python3"
+}
 
 // Name identifies the engine in the catalog.
 func (*SGLang) Name() string { return "sglang" }
@@ -55,10 +76,7 @@ func (g *SGLang) Commands(s ShardSpec) ([]Command, error) {
 }
 
 func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
-	python := g.cfg.Python
-	if python == "" {
-		python = "python3"
-	}
+	python := g.python()
 	p := s.Profile
 
 	args := []string{"-m", "sglang.launch_server",
@@ -78,6 +96,11 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
 		)
 	}
 	args = append(args, p.Args...)
+	// SGLang reads its key only from the command line, so on a node with
+	// other users it is visible to them in the process list.
+	if s.APIKey != "" && s.Leader() {
+		args = append(args, "--api-key", s.APIKey)
+	}
 
 	return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil
 }

internal/agent/engines/unsafeenv.go

@@ -0,0 +1,30 @@
+package engines
+
+import "strings"
+
+// UnsafeEnv reports whether a profile may not set an environment variable,
+// and why.
+//
+// A profile's env is the escape hatch for tuning no adapter can guess —
+// CUDA_*, NCCL_*, VLLM_*, HF_* — and it reaches an engine the agent starts as
+// root. Some variables are not tuning but a way to run other code in that
+// process: the dynamic loader's LD_PRELOAD, Python's PYTHONSTARTUP, a shell's
+// BASH_ENV. Set from a model definition, which an admin API or a config file
+// supplies, they would turn "add a model" into "run this as root on every
+// node". They are refused where a model is defined and again where an engine
+// starts.
+func UnsafeEnv(key string) (bool, string) {
+	k := strings.ToUpper(key)
+	switch {
+	case strings.HasPrefix(k, "LD_"), strings.HasPrefix(k, "DYLD_"):
+		return true, "it changes what the dynamic loader loads"
+	case k == "PATH":
+		return true, "it changes which binaries run"
+	case strings.HasPrefix(k, "PYTHON") && k != "PYTHONUNBUFFERED" && k != "PYTHONHASHSEED":
+		return true, "it changes what Python imports or runs at start"
+	case k == "BASH_ENV" || k == "ENV" || k == "PERL5OPT" || k == "PERL5LIB" ||
+		k == "NODE_OPTIONS" || k == "RUBYOPT" || k == "GCONV_PATH" || k == "OPENSSL_CONF":
+		return true, "it makes an interpreter or library load code"
+	}
+	return false, ""
+}

internal/agent/engines/vllm.go

@@ -3,24 +3,48 @@ package engines
 import (
 	"fmt"
 	"strconv"
+	"sync/atomic"
 )
 
 // VLLMConfig locates the vLLM installation and picks how it is launched.
 type VLLMConfig struct {
 	// Bin is the vllm executable.
 	Bin string `koanf:"bin"`
-	// LaunchMode selects the multi-node mechanism: "ray" or "mp".
+	// LaunchMode selects the multi-node mechanism: "mp", the default, or
+	// "ray".
 	//
-	// Which one works depends on the vLLM version, and the flags have changed
-	// more than once (plan §14). Both paths exist so a cluster can pick the one
-	// its version supports instead of waiting for a code change.
+	// mp is what vLLM documents today — every node runs serve with its rank,
+	// the workers headless — and needs nothing but vLLM. ray is the older way,
+	// kept for a vLLM too old for mp's multi-node flags; it needs Ray on every
+	// node and has not been checked against a real cluster.
 	LaunchMode string `koanf:"launch_mode"`
 	// RayBin is used when LaunchMode is "ray".
 	RayBin string `koanf:"ray_bin"`
 }
 
 // VLLM builds vLLM launch commands.
-type VLLM struct{ cfg VLLMConfig }
+type VLLM struct {
+	cfg       VLLMConfig
+	installed atomic.Pointer[string]
+}
+
+// UseInstalled points the engine at the environment the agent installed and
+// checked. It takes precedence over the configuration, as llama.cpp's does:
+// it is the one seen to run. An operator who wants their own sets the
+// control plane's engines.vllm to "".
+func (v *VLLM) UseInstalled(path string) { v.installed.Store(&path) }
+
+// bin is the vllm to run: the installed one, else the configured one, else
+// whatever PATH has.
+func (v *VLLM) bin() string {
+	if p := v.installed.Load(); p != nil {
+		return *p
+	}
+	if v.cfg.Bin != "" {
+		return v.cfg.Bin
+	}
+	return "vllm"
+}
 
 // Name identifies the engine in the catalog.
 func (*VLLM) Name() string { return "vllm" }
@@ -78,10 +102,7 @@ func (v *VLLM) Commands(s ShardSpec) ([]Command, error) {
 
 // serverCommand produces the vLLM server process.
 func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
-	bin := v.cfg.Bin
-	if bin == "" {
-		bin = "vllm"
-	}
+	bin := v.bin()
 	p := s.Profile
 
 	args := []string{"serve", p.ModelPath,
@@ -100,16 +121,27 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
 
 	if s.NNodes > 1 {
 		switch v.cfg.LaunchMode {
-		case "", "ray":
+		case "ray":
 			// Ray is started separately by the supervisor; vLLM then finds the
 			// cluster through the address in the environment.
 			env["VLLM_HOST_IP"] = s.LeaderAddr
 			env["RAY_ADDRESS"] = fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort)
 			args = append(args, "--distributed-executor-backend", "ray")
-		case "mp":
-			env["VLLM_HOST_IP"] = s.LeaderAddr
-			env["VLLM_PORT"] = strconv.Itoa(s.LeaderPort)
-			args = append(args, "--distributed-executor-backend", "mp")
+		case "", "mp":
+			// vLLM's own multi-node launch: every node runs the same serve
+			// with its rank and the leader's address, and the workers run
+			// headless — no API server, only their GPUs. VLLM_HOST_IP is
+			// this node's address for vLLM to find itself, so it is left
+			// to vLLM; the leader's, which workers used to be given, made
+			// every worker announce itself as the leader.
+			args = append(args, "--distributed-executor-backend", "mp",
+				"--nnodes", strconv.Itoa(s.NNodes),
+				"--node-rank", strconv.Itoa(s.Rank),
+				"--master-addr", s.LeaderAddr,
+				"--master-port", strconv.Itoa(s.LeaderPort))
+			if !s.Leader() {
+				args = append(args, "--headless")
+			}
 		default:
 			return Command{}, fmt.Errorf("vllm: unknown launch_mode %q (want \"ray\" or \"mp\")", v.cfg.LaunchMode)
 		}
@@ -118,13 +150,16 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
 	// Profile arguments last, so an operator can override anything above.
 	args = append(args, p.Args...)
 
+	if s.APIKey != "" {
+		env["VLLM_API_KEY"] = s.APIKey
+	}
 	return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil
 }
 
 // rayCommand returns the command that starts or joins the Ray cluster, or
 // ok=false when the launch mode does not use Ray.
 func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) {
-	if s.NNodes <= 1 || (v.cfg.LaunchMode != "" && v.cfg.LaunchMode != "ray") {
+	if s.NNodes <= 1 || v.cfg.LaunchMode != "ray" {
 		return Command{}, false
 	}
 	bin := v.cfg.RayBin
@@ -134,7 +169,10 @@ func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) {
 
 	args := []string{"start", "--block"}
 	if s.Leader() {
-		args = append(args, "--head", "--port", strconv.Itoa(s.LeaderPort))
+		// Ray's head has no authentication: whoever reaches its port can run
+		// code on the node. Its dashboard, and the job API behind it, is the
+		// easiest way in, and nothing here uses it.
+		args = append(args, "--head", "--port", strconv.Itoa(s.LeaderPort), "--include-dashboard=false")
 	} else {
 		args = append(args, "--address", fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort))
 	}

internal/agent/fabrics.go

@@ -0,0 +1,345 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
+)
+
+// usableFabrics reports the fabrics this node can actually carry traffic on.
+//
+// Loud about what it drops. A fabric silently missing from a registration looks
+// identical to one never configured, and the operator who typed the adapter
+// name is the only person who can fix it — so the reason is logged here, on the
+// machine where both halves are visible, rather than inferred later from a
+// replica that will not converge.
+func (a *Agent) usableFabrics(inv discovery.Inventory) []inventory.FabricAttachment {
+	out := a.checkFabrics(inv.HCAs, true)
+	a.mu.Lock()
+	a.fabricsSent = fabricsFingerprint(out)
+	a.mu.Unlock()
+	return out
+}
+
+// checkFabrics is the declared fabrics this node can carry traffic on, from
+// the adapters as they are now. Loud logs every problem; the heartbeat checks
+// quietly and speaks only when the answer changes.
+func (a *Agent) checkFabrics(hcas []discovery.HCA, loud bool) []inventory.FabricAttachment {
+	declared := map[string][]string{}
+	for _, f := range a.cfg.Fabrics {
+		declared[f.Fabric] = f.HCAs
+	}
+
+	var out []inventory.FabricAttachment
+	for _, c := range discovery.CheckFabrics(declared, hcas) {
+		if loud {
+			for _, p := range c.Problems {
+				a.log.Error("a declared fabric adapter cannot carry traffic",
+					"fabric", c.Fabric, "problem", p)
+			}
+		}
+		if !c.OK() {
+			if loud {
+				a.log.Error("fabric not advertised; no declared adapter on it can carry traffic",
+					"fabric", c.Fabric,
+					"consequence", "no multi-node replica will be placed here")
+			}
+			continue
+		}
+		out = append(out, inventory.FabricAttachment{
+			Fabric: c.Fabric, HCAs: c.Usable, RateGbps: c.RateGbps,
+		})
+	}
+	return out
+}
+
+// fabricsFingerprint is enough to notice a fabric, an adapter or a rate
+// changing.
+func fabricsFingerprint(fs []inventory.FabricAttachment) string {
+	var b strings.Builder
+	for _, f := range fs {
+		fmt.Fprintf(&b, "%s:%s:%d;", f.Fabric, strings.Join(f.HCAs, ","), f.RateGbps)
+	}
+	return b.String()
+}
+
+// sampleIB re-reads the adapters for the heartbeat: every port's state and
+// counters, and the fabrics again when they no longer match what the control
+// plane was told.
+//
+// Every heartbeat, because it is a few small files per port, and because a
+// port that went down used to be invisible until the agent restarted — the
+// node went on being offered multi-node work that would hang on its first
+// collective.
+func (a *Agent) sampleIB(hb *llmgwv1.Heartbeat) {
+	a.mu.Lock()
+	had := len(a.inv.HCAs) > 0
+	a.mu.Unlock()
+	if !had && len(a.cfg.Fabrics) == 0 {
+		return
+	}
+	hcas, err := discovery.DiscoverIB(a.cfg.Discovery.IBRoot)
+	if err != nil {
+		return
+	}
+	a.attachSM(hcas)
+	for _, h := range hcas {
+		for _, p := range h.Ports {
+			ip := &llmgwv1.IBPort{
+				Hca: h.Name, Port: int32(p.Num), State: p.State, PhysState: p.PhysState,
+				RateGbps: int32(p.RateGbps), LinkLayer: string(p.LinkLayer), Counters: p.Counters,
+				NodeGuid: h.NodeGUID, PortGuid: p.PortGUID, Network: p.Network, GidPrefix: p.GIDPrefix,
+			}
+			a.mu.Lock()
+			if p.SM != nil {
+				ip.SmGuid, ip.SmLid, ip.SmPriority, ip.SmState = p.SM.GUID, int32(p.SM.LID), int32(p.SM.Priority), p.SM.State
+			} else {
+				ip.SmError = a.smErr[smKey(h.Name, p.Num)]
+			}
+			for _, r := range a.sms[smKey(h.Name, p.Num)] {
+				ip.SubnetManagers = append(ip.SubnetManagers, &llmgwv1.SubnetManager{
+					Guid: r.GUID, Lid: int32(r.LID), Priority: int32(r.Priority), State: r.State})
+			}
+			ip.SubnetManagersError = a.smsErr[smKey(h.Name, p.Num)]
+			a.mu.Unlock()
+			hb.IbPorts = append(hb.IbPorts, ip)
+		}
+	}
+	a.mu.Lock()
+	a.inv.HCAs = hcas // a reconnect registers what is true now
+	a.mu.Unlock()
+	if len(a.cfg.Fabrics) == 0 {
+		return
+	}
+	now := a.checkFabrics(hcas, false)
+	fp := fabricsFingerprint(now)
+	a.mu.Lock()
+	changed := fp != a.fabricsSent
+	a.fabricsSent = fp
+	a.mu.Unlock()
+	if !changed {
+		return
+	}
+	// Said once, with the reasons, the moment it happens.
+	a.checkFabrics(hcas, true)
+	a.log.Warn("the fabrics this node can carry traffic on changed", "now", fp)
+	hb.FabricsChanged = true
+	for _, f := range now {
+		hb.Fabrics = append(hb.Fabrics, &llmgwv1.FabricAttachment{
+			Fabric: f.Fabric, Hcas: f.HCAs, RateGbps: int32(f.RateGbps),
+		})
+	}
+}
+
+func smKey(hca string, port int) string { return fmt.Sprintf("%s/%d", hca, port) }
+
+// scanFabric is discovery.ScanFabric, replaced in tests.
+var scanFabric = discovery.ScanFabric
+
+// querySMs is discovery.QuerySMs, replaced in tests.
+var querySMs = func(hca string, port int) ([]discovery.SMRecord, error) {
+	return discovery.QuerySMs(context.Background(), hca, port)
+}
+
+func splitSMKey(k string) (string, int) {
+	i := strings.LastIndexByte(k, '/')
+	port, _ := strconv.Atoi(k[i+1:])
+	return k[:i], port
+}
+
+// querySM is discovery.QuerySM, replaced in tests.
+var querySM = func(hca string, port int) (*discovery.SMInfo, error) {
+	return discovery.QuerySM(context.Background(), hca, port)
+}
+
+// attachSM puts on each active InfiniBand port the subnet manager it sees,
+// asking again once a minute and otherwise reusing the last answer.
+func (a *Agent) attachSM(hcas []discovery.HCA) {
+	a.mu.Lock()
+	stale := time.Since(a.smAsked) > time.Minute || a.sm == nil
+	a.mu.Unlock()
+	if stale {
+		sm, errs := map[string]*discovery.SMInfo{}, map[string]string{}
+		_ = discovery.AttachSM(hcas, func(hca string, port int) (*discovery.SMInfo, error) {
+			got, err := querySM(hca, port)
+			if err != nil {
+				errs[smKey(hca, port)] = err.Error()
+				return nil, err
+			}
+			sm[smKey(hca, port)] = got
+			return got, nil
+		})
+		// Every subnet manager on each fabric, through one port per master:
+		// the same list through every port of one fabric is one query too
+		// many for each.
+		sms, smsErr := map[string][]discovery.SMRecord{}, map[string]string{}
+		asked := map[string]bool{}
+		for k, v := range sm {
+			if asked[v.GUID] {
+				continue
+			}
+			asked[v.GUID] = true
+			hca, port := splitSMKey(k)
+			if recs, err := querySMs(hca, port); err != nil {
+				smsErr[k] = err.Error()
+			} else {
+				sms[k] = recs
+			}
+		}
+		a.mu.Lock()
+		for k, v := range sm {
+			if old := a.sm[k]; old != nil && old.GUID != v.GUID {
+				a.log.Warn("the master subnet manager changed", "port", k, "from", old.GUID, "to", v.GUID)
+			}
+		}
+		a.sm, a.smErr, a.smAsked = sm, errs, time.Now()
+		a.sms, a.smsErr = sms, smsErr
+		a.mu.Unlock()
+		return
+	}
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	for i := range hcas {
+		for j := range hcas[i].Ports {
+			if sm := a.sm[smKey(hcas[i].Name, hcas[i].Ports[j].Num)]; sm != nil && hcas[i].Ports[j].Active() {
+				hcas[i].Ports[j].SM = sm
+			}
+		}
+	}
+}
+
+// ensureDiags installs infiniband-diags on a node with adapters and without
+// it. sminfo is what tells which physical fabric a port is on; without it,
+// two separate fabrics given one name cannot be caught. Installed the way
+// engine dependencies are, from the machine's own package sources.
+func (a *Agent) ensureDiags(ctx context.Context) {
+	a.mu.Lock()
+	hasIB := len(a.inv.HCAs) > 0
+	a.mu.Unlock()
+	if !hasIB || a.prov == nil {
+		return
+	}
+	if _, err := exec.LookPath("sminfo"); err == nil {
+		return
+	}
+	a.log.Info("installing infiniband-diags, to tell which fabric each port is on")
+	pkgs := map[string][]string{"deb": {"infiniband-diags"}, "rpm": {"infiniband-diags"}}
+	if err := a.prov.InstallPackages(ctx, pkgs); err != nil {
+		a.log.Warn("could not install infiniband-diags; fabrics cannot be told apart by their subnet manager",
+			"err", err)
+	}
+}
+
+// FeatureFabricTest is the agent answering FabricTestRequest.
+const FeatureFabricTest = "fabric-test"
+
+// runWriteBW is discovery.RunWriteBW, and haveWriteBW whether it is
+// installed; both replaced in tests.
+var (
+	runWriteBW  = discovery.RunWriteBW
+	haveWriteBW = func() bool { _, err := exec.LookPath("ib_write_bw"); return err == nil }
+)
+
+// fabricTest runs this node's side of a bandwidth test with another.
+func (a *Agent) fabricTest(ctx context.Context, req *llmgwv1.FabricTestRequest) *llmgwv1.FabricTestReply {
+	start := time.Now()
+	reply := &llmgwv1.FabricTestReply{RequestId: req.GetRequestId()}
+	defer func() { reply.TookMs = time.Since(start).Milliseconds() }()
+
+	w := discovery.WriteBW{HCA: req.GetHca(), Port: int(req.GetPort()), GIDIndex: -1,
+		TCPPort: int(req.GetTcpPort()), Seconds: int(req.GetSeconds())}
+	if req.GetRole() == "client" {
+		w.Peer = req.GetPeer()
+	}
+	a.mu.Lock()
+	found := false
+	for _, h := range a.inv.HCAs {
+		for _, p := range h.Ports {
+			if h.Name == w.HCA && p.Num == w.Port {
+				found = true
+				// RoCE needs its v2 GID named, as NCCL does; InfiniBand uses the LID.
+				if p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID >= 0 {
+					w.GIDIndex = p.RoCEv2GID
+				}
+			}
+		}
+	}
+	a.mu.Unlock()
+	if !found {
+		reply.Error = fmt.Sprintf("this node has no port %s/%d", w.HCA, w.Port)
+		return reply
+	}
+	if !haveWriteBW() && a.prov != nil {
+		a.log.Info("installing perftest, for a bandwidth test with another node")
+		pkgs := map[string][]string{"deb": {"perftest"}, "rpm": {"perftest"}}
+		if err := a.prov.InstallPackages(ctx, pkgs); err != nil {
+			reply.Error = "installing perftest: " + err.Error()
+			return reply
+		}
+	}
+	gbps, out, err := runWriteBW(ctx, w)
+	reply.Gbps = gbps
+	reply.Output = tailLines(out, 12)
+	if err != nil {
+		reply.Error = err.Error()
+	}
+	return reply
+}
+
+func tailLines(s string, n int) string {
+	lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
+	if len(lines) > n {
+		lines = lines[len(lines)-n:]
+	}
+	return strings.Join(lines, "\n")
+}
+
+// discoverGPUFabric and discoverIMEX are replaced in tests.
+var (
+	discoverGPUFabric = discovery.DiscoverGPUFabric
+	discoverIMEX      = discovery.DiscoverIMEX
+)
+
+// sampleNVLink reads, once a minute, which NVLink domain each GPU is joined
+// to and whether IMEX is up. nvidia-smi -q takes a second or two, too long
+// for every heartbeat, and a fabric that was joined an hour ago rarely
+// changes in five seconds — but when a fabric manager restarts it does, for
+// every card at once.
+func (a *Agent) sampleNVLink(hb *llmgwv1.Heartbeat) {
+	a.mu.Lock()
+	gpus := slices.Clone(a.inv.GPUs)
+	due := time.Since(a.nvlAsked) > time.Minute
+	if due {
+		a.nvlAsked = time.Now()
+	}
+	a.mu.Unlock()
+	if len(gpus) == 0 || !due {
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	fab, err := discoverGPUFabric(ctx, a.cfg.Discovery.NvidiaSMI)
+	if err != nil || len(fab) == 0 {
+		return
+	}
+	discovery.ApplyFabric(gpus, fab)
+	for _, g := range gpus {
+		if g.Fabric == nil {
+			continue
+		}
+		hb.GpuFabric = append(hb.GpuFabric, &llmgwv1.GPUFabricState{
+			Index: int32(g.Index), State: g.Fabric.State, Status: g.Fabric.Status,
+			ClusterUuid: g.Fabric.ClusterUUID, CliqueId: g.Fabric.CliqueID, Bandwidth: g.Fabric.Bandwidth,
+		})
+	}
+	m := discoverIMEX(ctx)
+	hb.Imex = &llmgwv1.IMEX{State: m.State, Channels: int32(m.Channels)}
+}

internal/agent/gpucheck.go

@@ -0,0 +1,155 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+)
+
+// gpuProcesses and processGroup are variables so tests can stand in for
+// nvidia-smi and /proc.
+var (
+	gpuProcesses = discovery.GPUProcesses
+	gpuMemory    = discovery.GPUMemoryByProcess
+	processGroup = pgidOf
+	// gpuCheckWait is how long a shard has, once it answers, to show up on
+	// its GPUs: the weights are on them by then, but nvidia-smi can lag.
+	gpuCheckWait = 15 * time.Second
+)
+
+// onItsGPUs checks that a shard placed on GPUs computes on them.
+//
+// For an engine that falls back to the CPU when its GPU backend does not load,
+// "it answers" is not "it is ready": llama.cpp built for CUDA but missing
+// cuBLAS comes up on the CPU, answers at a tenth of the speed, and the GPU it
+// was given sits idle and reserved. What it computes on is what nvidia-smi
+// lists it on, so a shard none of whose processes hold memory on one of its
+// GPUs is refused, with what the engine itself can see.
+func (a *Agent) onItsGPUs(ctx context.Context, eng engines.Engine, spec engines.ShardSpec, pid int) error {
+	fb, ok := eng.(engines.CPUFallback)
+	if !ok || !fb.FallsBackToCPU() || len(spec.GPUIndices) == 0 || pid <= 0 {
+		return nil
+	}
+	group := processGroup(pid)
+	deadline := time.Now().Add(gpuCheckWait)
+	var missing []int
+	for {
+		procs, err := gpuProcesses(ctx, a.cfg.Discovery.NvidiaSMI)
+		if err != nil || procs == nil {
+			// No nvidia-smi to ask is no evidence either way, and a node that
+			// was given GPUs has one; say so and let the shard serve.
+			a.log.Warn("could not check that a GPU shard runs on its GPUs", "shard", spec.ShardID, "err", err)
+			return nil
+		}
+		missing = missing[:0]
+		for _, i := range spec.GPUIndices {
+			if !holds(procs[i], pid, group) {
+				missing = append(missing, i)
+			}
+		}
+		if len(missing) == 0 {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(time.Second):
+		}
+	}
+	seen := "it lists no GPU at all"
+	if devs, err := fb.Devices(ctx); err != nil {
+		seen = "listing its devices failed: " + err.Error()
+	} else if len(devs) > 0 {
+		seen = "it lists " + strings.Join(devs, "; ")
+	}
+	return fmt.Errorf("placed on GPU %s but computing on the CPU: the engine holds no memory there. "+
+		"Its GPU backend did not load (%s). Check the build with `ldd` on its libggml-cuda.so, "+
+		"or run `llmgw doctor` on this node", joinIndices(missing), seen)
+}
+
+// foreignGPUMemory is what each GPU holds for anything other than this
+// agent's shards, by index, and each GPU's memory in use. Nil when nvidia-smi
+// cannot be asked, which is no reading rather than an empty card.
+//
+// Everything allocated less what the shards' processes hold, rather than the
+// sum of the other processes: a process in another container's PID namespace
+// holds memory nvidia-smi lists under no PID this node can see, and it is
+// still not memory placement can hand out.
+func (a *Agent) foreignGPUMemory(ctx context.Context, shards []int) (foreign, used []int32) {
+	mem, err := gpuMemory(ctx, a.cfg.Discovery.NvidiaSMI)
+	if err != nil || mem == nil {
+		return nil, nil
+	}
+	groups := make([]int, len(shards))
+	for i, pid := range shards {
+		groups[i] = processGroup(pid)
+	}
+	ours := func(p int) bool {
+		for i, pid := range shards {
+			if holds([]int{p}, pid, groups[i]) {
+				return true
+			}
+		}
+		return false
+	}
+	for _, g := range mem {
+		f := g.UsedMB
+		for _, p := range g.Procs {
+			if ours(p.PID) {
+				f -= p.UsedMB
+			}
+		}
+		foreign = append(foreign, int32(max(f, 0)))
+		used = append(used, int32(g.UsedMB))
+	}
+	return foreign, used
+}
+
+// holds reports whether any of pids is the shard's process or in its group:
+// an engine that computes in a child process is still the shard.
+func holds(pids []int, pid, group int) bool {
+	for _, p := range pids {
+		if p == pid || (group > 0 && processGroup(p) == group) {
+			return true
+		}
+	}
+	return false
+}
+
+// pgidOf reads a process's group from /proc; zero when it cannot.
+func pgidOf(pid int) int {
+	b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat")
+	if err != nil {
+		return 0
+	}
+	// The command name, in parentheses, may hold spaces; the fields after its
+	// closing parenthesis are state, ppid, pgrp.
+	s := string(b)
+	i := strings.LastIndexByte(s, ')')
+	if i < 0 {
+		return 0
+	}
+	f := strings.Fields(s[i+1:])
+	if len(f) < 3 {
+		return 0
+	}
+	n, _ := strconv.Atoi(f[2])
+	return n
+}
+
+func joinIndices(is []int) string {
+	s := make([]string, len(is))
+	for k, i := range is {
+		s[k] = strconv.Itoa(i)
+	}
+	return strings.Join(s, ", ")
+}

internal/agent/gpucheck_test.go

@@ -0,0 +1,124 @@
+package agent
+
+import (
+	"context"
+	"log/slog"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+)
+
+// fallbackEngine is an engine that comes up on the CPU when its GPU backend
+// does not load, and lists the devices it can see.
+type fallbackEngine struct {
+	engines.Engine
+	devices []string
+}
+
+func (fallbackEngine) FallsBackToCPU() bool { return true }
+func (f fallbackEngine) Devices(context.Context) ([]string, error) {
+	return f.devices, nil
+}
+
+func TestAGPUShardComputingOnTheCPUIsNotReady(t *testing.T) {
+	procs := map[int][]int{}
+	oldProcs, oldGroup, oldWait := gpuProcesses, processGroup, gpuCheckWait
+	gpuProcesses = func(context.Context, string) (map[int][]int, error) { return procs, nil }
+	processGroup = func(pid int) int {
+		if pid >= 500 && pid < 600 {
+			return 500 // the shard's group: 500 and its children
+		}
+		return pid
+	}
+	gpuCheckWait = 0
+	t.Cleanup(func() { gpuProcesses, processGroup, gpuCheckWait = oldProcs, oldGroup, oldWait })
+
+	a := &Agent{log: slog.Default()}
+	spec := engines.ShardSpec{ShardID: "s", GPUIndices: []int{0, 1}}
+	eng := fallbackEngine{}
+
+	// On no GPU at all: refused, saying what the engine sees.
+	err := a.onItsGPUs(context.Background(), eng, spec, 500)
+	if err == nil || !strings.Contains(err.Error(), "computing on the CPU") || !strings.Contains(err.Error(), "lists no GPU") {
+		t.Fatalf("a shard on no GPU was let through: %v", err)
+	}
+	// On one of its two: refused, naming the one it is missing.
+	procs[0] = []int{500}
+	if err := a.onItsGPUs(context.Background(), eng, spec, 500); err == nil || !strings.Contains(err.Error(), "GPU 1 ") {
+		t.Fatalf("a shard on half its GPUs: %v", err)
+	}
+	// Another process on GPU 1 is not the shard.
+	procs[1] = []int{900}
+	if err := a.onItsGPUs(context.Background(), eng, spec, 500); err == nil {
+		t.Fatal("another process's memory was taken for the shard's")
+	}
+	// A child of the shard holding GPU 1 is the shard.
+	procs[1] = []int{900, 512}
+	if err := a.onItsGPUs(context.Background(), eng, spec, 500); err != nil {
+		t.Fatalf("a shard on both its GPUs: %v", err)
+	}
+
+	// An engine that fails rather than falling back is not checked, nor is a
+	// shard with no GPUs.
+	procs = map[int][]int{}
+	if err := a.onItsGPUs(context.Background(), struct{ engines.Engine }{}, spec, 500); err != nil {
+		t.Errorf("checked an engine that does not fall back: %v", err)
+	}
+	if err := a.onItsGPUs(context.Background(), eng, engines.ShardSpec{}, 500); err != nil {
+		t.Errorf("checked a CPU shard: %v", err)
+	}
+}
+
+func TestTheCheckWaitsForTheShardToShowUpOnItsGPU(t *testing.T) {
+	calls := 0
+	oldProcs, oldWait := gpuProcesses, gpuCheckWait
+	gpuProcesses = func(context.Context, string) (map[int][]int, error) {
+		calls++
+		if calls < 2 {
+			return map[int][]int{}, nil
+		}
+		return map[int][]int{0: {77}}, nil
+	}
+	gpuCheckWait = 5 * time.Second
+	t.Cleanup(func() { gpuProcesses, gpuCheckWait = oldProcs, oldWait })
+
+	a := &Agent{log: slog.Default()}
+	if err := a.onItsGPUs(context.Background(), fallbackEngine{}, engines.ShardSpec{GPUIndices: []int{0}}, 77); err != nil {
+		t.Fatalf("nvidia-smi lagging by a second failed the shard: %v", err)
+	}
+}
+
+func TestForeignGPUMemoryLeavesOutTheAgentsOwnShards(t *testing.T) {
+	// cuda01: ComfyUI and one of our engines on the same card. Only ComfyUI's
+	// share — and whatever no listed process accounts for — is foreign.
+	oldMem, oldGroup := gpuMemory, processGroup
+	gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) {
+		return []discovery.GPUMemory{
+			{UsedMB: 15000, Procs: []discovery.GPUProcess{{PID: 71315, UsedMB: 4932}, {PID: 512, UsedMB: 9500}}},
+			{UsedMB: 300},
+		}, nil
+	}
+	processGroup = func(pid int) int {
+		if pid >= 500 && pid < 600 {
+			return 500
+		}
+		return pid
+	}
+	t.Cleanup(func() { gpuMemory, processGroup = oldMem, oldGroup })
+
+	a := &Agent{log: slog.Default()}
+	foreign, used := a.foreignGPUMemory(context.Background(), []int{500})
+	if !slices.Equal(foreign, []int32{5500, 300}) || !slices.Equal(used, []int32{15000, 300}) {
+		t.Errorf("foreign %v, used %v; want [5500 300] and [15000 300]", foreign, used)
+	}
+
+	// No nvidia-smi to ask is no reading, not an empty card.
+	gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) { return nil, nil }
+	if foreign, _ := a.foreignGPUMemory(context.Background(), nil); foreign != nil {
+		t.Errorf("no reading came back as %v", foreign)
+	}
+}

internal/agent/logs.go

@@ -0,0 +1,34 @@
+package agent
+
+import (
+	"slices"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+)
+
+// logsReply answers a request for one of this node's logs.
+func (a *Agent) logsReply(req *llmgwv1.LogsRequest) *llmgwv1.LogsReply {
+	n := int(req.GetLines())
+	if n <= 0 {
+		n = 200
+	}
+	n = min(n, agentLogLines)
+	shards := a.sup.IDs()
+	slices.Sort(shards)
+	out := &llmgwv1.LogsReply{
+		RequestId: req.GetRequestId(),
+		Sources:   append([]string{"agent"}, shards...),
+	}
+	switch src := req.GetSource(); src {
+	case "", "agent":
+		out.Lines = a.logRing.tail(n)
+	default:
+		sh, err := a.sup.Get(src)
+		if err != nil {
+			out.Error = "no engine " + src + " runs on this node"
+			return out
+		}
+		out.Lines = sh.Logs(n)
+	}
+	return out
+}

internal/agent/provision.go

@@ -0,0 +1,227 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"maps"
+	"slices"
+	"strings"
+	"time"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
+)
+
+// engines is what this node can run: what the agent installed and saw run,
+// and what the configuration names and answers. Nothing merely found on the
+// machine — a llama-server on PATH is somebody's, not necessarily one that
+// works, and placing on it is how a replica fails on a node nobody watches.
+func (a *Agent) engines(ctx context.Context) map[string]string {
+	a.engineOnce.Do(func() {
+		vers := a.reg.ProbeConfigured(ctx)
+		a.mu.Lock()
+		a.configuredVers, a.lastEngineProbe = vers, time.Now()
+		a.mu.Unlock()
+		if len(vers) > 0 {
+			a.log.Info("engines named in the configuration",
+				"engines", engines.ProbedNames(vers))
+		}
+	})
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	vers := a.engineReportLocked()
+	// Whatever registration sends is what the control plane now knows, so a
+	// heartbeat repeats it only once it differs.
+	a.enginesSent = enginesFingerprint(vers)
+	return vers
+}
+
+// engineReportLocked merges the two sources. Installed wins: it is the one
+// that was seen to run. Called with mu held.
+func (a *Agent) engineReportLocked() map[string]string {
+	out := maps.Clone(a.configuredVers)
+	if out == nil {
+		out = map[string]string{}
+	}
+	maps.Copy(out, a.installedVers)
+	maps.DeleteFunc(out, func(e, _ string) bool { return a.disabled(e) })
+	return out
+}
+
+// byHand reports whether agent.yaml names this engine's binary, which opts
+// it out of being installed.
+func (a *Agent) byHand(engine string) bool {
+	if a.reg == nil {
+		return false
+	}
+	e, err := a.reg.Get(engine)
+	if err != nil {
+		return false
+	}
+	c, ok := e.(engines.Configured)
+	return ok && c.Configured()
+}
+
+// disabled reports whether agent.yaml says this node never runs an engine:
+// not installed when asked, and not advertised when found.
+func (a *Agent) disabled(engine string) bool {
+	return slices.Contains(a.cfg.Engines.Disabled, engine)
+}
+
+// provisionEvery is how often installed engines are checked again. A library
+// removed by an unrelated upgrade is found here rather than by the next
+// replica to start.
+const provisionEvery = 10 * time.Minute
+
+// setProvision takes the control plane's list of wanted engines.
+func (a *Agent) setProvision(p *llmgwv1.Provision) {
+	wants := []provision.Want{}
+	for _, e := range p.GetEngines() {
+		if a.disabled(e.GetEngine()) {
+			continue // a control plane older than the node's say in it
+		}
+		if a.byHand(e.GetEngine()) {
+			// The operator named a binary for it: a local build, a CUDA
+			// runtime of their own. Installing one beside it would win —
+			// the installed build is used first — and quietly replace theirs.
+			continue
+		}
+		wants = append(wants, provision.Want{Engine: e.GetEngine(), Version: e.GetVersion(), Variant: e.GetVariant()})
+	}
+	a.mu.Lock()
+	a.provWants = wants
+	a.mu.Unlock()
+	select {
+	case a.provKick <- struct{}{}:
+	default:
+	}
+}
+
+// provisionLoop installs and re-checks the wanted engines, one pass at a time.
+func (a *Agent) provisionLoop(ctx context.Context) {
+	// After a failure the next pass comes sooner, and backs off: a lock held
+	// by a new machine's own updates clears in a minute or two, while ten
+	// minutes' wait left a node idle for no reason.
+	next := provisionEvery
+	retry := time.Minute
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-a.provKick:
+		case <-time.After(next):
+		}
+		a.mu.Lock()
+		wants := slices.Clone(a.provWants)
+		managed := a.provWants != nil
+		a.mu.Unlock()
+		if !managed {
+			continue
+		}
+
+		statuses := a.prov.Converge(ctx, wants)
+		next = provisionEvery
+		for _, st := range statuses {
+			if st.State == provision.Failed {
+				next, retry = retry, min(retry*2, provisionEvery)
+				break
+			}
+		}
+		if next == provisionEvery {
+			retry = time.Minute
+		}
+		installed := map[string]string{}
+		keep := map[string]bool{}
+		for _, st := range statuses {
+			keep[st.Engine] = true
+			if st.State == provision.Ready {
+				a.reg.UseInstalled(st.Engine, st.Binary)
+				installed[st.Engine] = st.Version
+			}
+		}
+		a.mu.Lock()
+		if enginesFingerprint(installed) != enginesFingerprint(a.installedVers) {
+			a.log.Info("engines installed and running", "engines", engines.ProbedNames(installed))
+		}
+		a.installedVers = installed
+		// An engine no longer asked for is no longer reported on.
+		for e := range a.provStatus {
+			if !keep[e] {
+				delete(a.provStatus, e)
+			}
+		}
+		a.mu.Unlock()
+	}
+}
+
+// provisionStatus records a change in one engine's installation.
+func (a *Agent) provisionStatus(st provision.Status) {
+	a.mu.Lock()
+	prev, had := a.provStatus[st.Engine]
+	a.provStatus[st.Engine] = st
+	a.mu.Unlock()
+	if !had || prev.State != st.State || prev.Detail != st.Detail {
+		lvl := slog.LevelInfo
+		if st.State == provision.Failed || st.State == provision.Unsupported {
+			lvl = slog.LevelWarn
+		}
+		a.log.Log(context.Background(), lvl, "engine installation",
+			"engine", st.Engine, "version", st.Version, "variant", st.Variant,
+			"state", st.State, "detail", st.Detail)
+	}
+}
+
+func provisionFingerprint(m map[string]provision.Status) string {
+	var b strings.Builder
+	for _, k := range slices.Sorted(maps.Keys(m)) {
+		st := m[k]
+		fmt.Fprintf(&b, "%s|%s|%s|%s|%s\n", st.Engine, st.Version, st.Variant, st.State, st.Detail)
+	}
+	return b.String()
+}
+
+// engineReprobe is how often the installed engines are looked for again.
+//
+// Minutes, not seconds: an engine is installed by somebody at a keyboard, so a
+// minute's delay in noticing is invisible, while SGLang's probe starts a
+// Python interpreter that imports it.
+const engineReprobe = time.Minute
+
+// reprobeEngines looks for installed engines again when it is time, off the
+// heartbeat goroutine so a slow probe never delays a heartbeat.
+func (a *Agent) reprobeEngines() {
+	a.mu.Lock()
+	due := !a.probingEngines && !a.lastEngineProbe.IsZero() &&
+		time.Since(a.lastEngineProbe) >= engineReprobe
+	if due {
+		a.probingEngines = true
+	}
+	a.mu.Unlock()
+	if !due {
+		return
+	}
+	go func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		vers := a.reg.ProbeConfigured(ctx)
+
+		a.mu.Lock()
+		changed := enginesFingerprint(vers) != enginesFingerprint(a.configuredVers)
+		a.configuredVers, a.lastEngineProbe, a.probingEngines = vers, time.Now(), false
+		a.mu.Unlock()
+		if changed {
+			a.log.Info("engines named in the configuration changed",
+				"engines", engines.ProbedNames(vers))
+		}
+	}()
+}
+
+func enginesFingerprint(vers map[string]string) string {
+	var b strings.Builder
+	for _, k := range slices.Sorted(maps.Keys(vers)) {
+		fmt.Fprintf(&b, "%s=%s\n", k, vers[k])
+	}
+	return b.String()
+}

internal/agent/provision/install.go

@@ -16,6 +16,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -26,6 +27,9 @@ import (
 type Want struct {
 	Engine  string
 	Version string
+	// Variant is a build chosen for this node ("cu129"); empty leaves it to
+	// the node's hardware.
+	Variant string
 }
 
 // The states an engine's installation can be in.
@@ -34,6 +38,9 @@ const (
 	Ready       = "ready"
 	Failed      = "failed"
 	Unsupported = "unsupported"
+	// Skipped is an engine this node is not meant to run, such as vLLM on a
+	// machine with no GPU. Nothing is wrong, and nothing is shown as wrong.
+	Skipped = "skipped"
 )
 
 // Status is where one engine's installation stands.
@@ -62,11 +69,22 @@ type Installer struct {
 
 	// Run executes a command; a seam for tests. Nil runs it for real.
 	Run func(ctx context.Context, env []string, name string, args ...string) ([]byte, error)
-	// APIBase and DownloadBase default to GitHub; tests point them at a fake.
+	// FreeSpace reports the gigabytes free under a directory; a seam for
+	// tests. Nil asks the filesystem.
+	FreeSpace func(dir string) (float64, error)
+	// RunStdin is Run with input; a seam for tests. Nil runs it for real.
+	RunStdin func(ctx context.Context, env []string, stdin, name string, args ...string) ([]byte, error)
+	// APIBase and DownloadBase default to GitHub, PyPIBase to PyPI; tests
+	// point them at a fake.
 	APIBase      string
 	DownloadBase string
+	PyPIBase     string
 }
 
+// validVersion is what a version may look like: "b10991", "0.29.0",
+// "0.11.0+cu128". No separators, and not starting with a dot.
+var validVersion = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$`)
+
 // DefaultRoot is where engines are installed when nothing says otherwise.
 const DefaultRoot = "/opt/llmgw/engines"
 
@@ -75,12 +93,16 @@ const DefaultRoot = "/opt/llmgw/engines"
 const marker = ".llmgw-installed"
 
 type markerFile struct {
-	Engine    string            `json:"engine"`
-	Version   string            `json:"version"`
-	Variant   string            `json:"variant"`
-	Binary    string            `json:"binary"`
-	SHA256    map[string]string `json:"sha256"`
-	Installed time.Time         `json:"installed"`
+	Engine  string            `json:"engine"`
+	Version string            `json:"version"`
+	Variant string            `json:"variant"`
+	Binary  string            `json:"binary"`
+	SHA256  map[string]string `json:"sha256,omitempty"`
+	// For a Python engine: what was asked for, PyPI as of when, by which uv.
+	Requirements []string  `json:"requirements,omitempty"`
+	ExcludeNewer string    `json:"exclude_newer,omitempty"`
+	UV           string    `json:"uv,omitempty"`
+	Installed    time.Time `json:"installed"`
 }
 
 // Converge brings every wanted engine to ready, or says why it cannot.
@@ -100,6 +122,15 @@ func (in *Installer) Converge(ctx context.Context, wants []Want) []Status {
 
 func (in *Installer) one(ctx context.Context, w Want) Status {
 	st := Status{Engine: w.Engine, Version: w.Version}
+	// Checked here, on the node, whatever the control plane checked: the
+	// version names a directory under the install root and is passed to a
+	// package manager, and "../x" would put an install — and the RemoveAll
+	// that replaces one — outside it, as root.
+	if !validVersion.MatchString(w.Version) {
+		st.State = Unsupported
+		st.Detail = fmt.Sprintf("%q is not a version this agent will install", w.Version)
+		return st
+	}
 	recipe, ok := Recipes[w.Engine]
 	if !ok {
 		st.State = Unsupported
@@ -107,19 +138,32 @@ func (in *Installer) one(ctx context.Context, w Want) Status {
 			"; install it by hand and name it in the agent's configuration"
 		return st
 	}
-	build, err := recipe.Choose(w.Version, in.Host)
-	if err != nil {
-		st.State, st.Detail = Unsupported, err.Error()
+	if w.Variant != "" && !validVersion.MatchString(w.Variant) {
+		st.State = Unsupported
+		st.Detail = fmt.Sprintf("%q is not a build this agent will install", w.Variant)
+		return st
+	}
+	builds, err := recipe.Candidates(w.Version, w.Variant, in.Host)
+	if err != nil {
+		st.State, st.Detail = Unsupported, err.Error()
+		var nh NotHere
+		if errors.As(err, &nh) {
+			st.State = Skipped
+		}
 		return st
 	}
-	st.Variant = build.Variant
-	dir := filepath.Join(in.root(), w.Engine, w.Version+"-"+build.Variant)
 
-	// Installed before: check it still runs, and trust nothing else about it.
-	if _, err := os.Stat(filepath.Join(dir, marker)); err == nil {
+	// Installed before: whichever candidate is, best first, checked that it
+	// still runs and trusted for nothing else.
+	for _, build := range builds {
+		dir := in.dirFor(w, build)
+		if _, err := os.Stat(filepath.Join(dir, marker)); err != nil {
+			continue
+		}
+		st := st.as(w, build)
 		// The packages are checked too. Removing one is exactly the kind of
-		// change an unrelated upgrade makes, and ldd below would catch it only
-		// as a missing library with no hint of which package held it.
+		// change an unrelated upgrade makes, and ldd below would catch it
+		// only as a missing library with no hint of which package held it.
 		if err := in.packages(ctx, build.Packages, &st); err != nil {
 			st.State, st.Detail = Failed, err.Error()
 			return st
@@ -129,24 +173,64 @@ func (in *Installer) one(ctx context.Context, w Want) Status {
 			return in.ready(st, build, bin, version)
 		}
 		in.log().Warn("an installed engine no longer runs; installing it again",
-			"engine", w.Engine, "version", w.Version, "err", err)
+			"engine", w.Engine, "version", st.Version, "variant", build.Variant, "err", err)
+		break
 	}
 
-	if err := in.install(ctx, recipe, w, build, dir, &st); err != nil {
-		st.State, st.Detail, st.Binary = Failed, err.Error(), ""
-		return st
+	// Each candidate in turn until one installs and is seen to work: a build
+	// the table was wrong about — a driver too old for it after all, a GPU
+	// it has no kernels for — costs its download, and the next is tried.
+	var tried []string
+	for _, build := range builds {
+		st := st.as(w, build)
+		if err := in.install(ctx, recipe, w, build, in.dirFor(w, build), &st); err != nil {
+			tried = append(tried, build.Variant+": "+err.Error())
+			in.log().Warn("an engine build did not work here; trying the next",
+				"engine", w.Engine, "variant", build.Variant, "err", err)
+			continue
+		}
+		bin, version, err := in.verify(ctx, in.dirFor(w, build), build)
+		if err != nil {
+			tried = append(tried, build.Variant+": "+err.Error())
+			continue
+		}
+		return in.ready(st, build, bin, version)
 	}
-	bin, version, err := in.verify(ctx, dir, build)
-	if err != nil {
-		st.State, st.Detail = Failed, err.Error()
-		return st
+	st.State, st.Binary = Failed, ""
+	st.Detail = strings.Join(tried, "; then ")
+	if len(builds) == 1 {
+		_, st.Detail, _ = strings.Cut(st.Detail, ": ")
 	}
-	return in.ready(st, build, bin, version)
+	return st
+}
+
+// dirFor is where a build of an engine is installed: by the version it is
+// and its variant, so two builds of one version sit side by side.
+func (in *Installer) dirFor(w Want, b Build) string {
+	return filepath.Join(in.root(), w.Engine, b.versionOr(w.Version)+"-"+b.Variant)
+}
+
+// versionOr is the version a build is: its own, when it is not the one asked
+// for.
+func (b Build) versionOr(asked string) string {
+	if b.Version != "" {
+		return b.Version
+	}
+	return asked
+}
+
+// as is a status for one build of what was asked for.
+func (st Status) as(w Want, b Build) Status {
+	st.Variant, st.Version = b.Variant, b.versionOr(w.Version)
+	return st
 }
 
 func (in *Installer) ready(st Status, b Build, bin, version string) Status {
 	st.State, st.Binary = Ready, bin
 	st.Detail = strings.TrimSpace(filepath.Base(bin) + " " + version)
+	if b.Note != "" {
+		st.Detail += "; " + b.Note
+	}
 	var missing []string
 	for _, f := range b.CPUFlags {
 		if !in.Host.HasCPUFlag(f) {
@@ -168,6 +252,9 @@ func (in *Installer) ready(st Status, b Build, bin, version string) Status {
 // Into a scratch directory renamed into place at the end, so a failure halfway
 // leaves nothing that looks installed.
 func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error {
+	if b.Python != "" {
+		return in.installPython(ctx, r, w, b, dir, st)
+	}
 	if err := in.packages(ctx, b.Packages, st); err != nil {
 		return err
 	}
@@ -235,24 +322,84 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir
 // running it is the only proof that the rest — the right instruction set, the
 // right driver — is in place.
 func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, version string, err error) {
-	bin, err = findFile(dir, b.Binary)
-	if err != nil {
-		return "", "", err
-	}
-	if out, err := in.run(ctx, nil, "ldd", bin); err == nil {
-		if missing := lddMissing(string(out)); len(missing) > 0 {
-			return "", "", fmt.Errorf("%s cannot run on this system: %s", b.Binary, strings.Join(missing, "; "))
+	timeout := 60 * time.Second
+	if b.Python != "" {
+		// A script in the environment, not a binary to ldd; and importing
+		// torch and the engine behind it takes a while on a cold disk.
+		bin, timeout = filepath.Join(dir, "bin", b.Binary), 3*time.Minute
+		if _, err := os.Stat(bin); err != nil {
+			return "", "", fmt.Errorf("%s is not in the environment: %w", b.Binary, err)
+		}
+	} else {
+		bin, err = findFile(dir, b.Binary)
+		if err != nil {
+			return "", "", err
+		}
+		if out, err := in.run(ctx, nil, "ldd", bin); err == nil {
+			if missing := lddMissing(string(out)); len(missing) > 0 {
+				return "", "", fmt.Errorf("%s cannot run on this system: %s", b.Binary, strings.Join(missing, "; "))
+			}
 		}
 	}
-	tctx, cancel := context.WithTimeout(ctx, 60*time.Second)
+	args := b.VersionArgs
+	if len(args) == 0 {
+		args = []string{"--version"}
+	}
+	tctx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
-	out, err := in.run(tctx, nil, bin, "--version")
+	out, err := in.run(tctx, nil, bin, args...)
 	if err != nil {
 		return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out))
 	}
+	if b.GPUBackend != "" {
+		if err := in.verifyGPUBackend(tctx, dir, bin, b); err != nil {
+			return "", "", err
+		}
+	}
 	return bin, firstVersionLine(out), nil
 }
 
+// verifyGPUBackend checks a GPU build can compute on this node's GPUs.
+//
+// The binary's own ldd says nothing about it: the GPU backend is a library
+// the server opens at run time, and when it cannot — cuBLAS missing, a driver
+// too old — the server carries on with its CPU backend. So the build is asked
+// which devices it sees, and when it sees none, ldd on the backend says why.
+func (in *Installer) verifyGPUBackend(ctx context.Context, dir, bin string, b Build) error {
+	out, err := in.run(ctx, nil, bin, "--list-devices")
+	if err != nil {
+		return fmt.Errorf("%s --list-devices failed: %v: %s", b.Binary, err, lastLine(out))
+	}
+	if listsDevice(string(out)) {
+		return nil
+	}
+	why := "its GPU backend did not load"
+	if lib, err := findFile(dir, b.GPUBackend); err != nil {
+		why = "the build has no " + b.GPUBackend
+	} else if out, err := in.run(ctx, nil, "ldd", lib); err == nil {
+		if missing := lddMissing(string(out)); len(missing) > 0 {
+			why = b.GPUBackend + " cannot load: " + strings.Join(missing, "; ")
+		}
+	}
+	return fmt.Errorf("the %s build of %s sees no GPU, and would run on the CPU: %s", b.Variant, b.Binary, why)
+}
+
+// listsDevice reads llama-server --list-devices: a device line after
+// "Available devices:", which holds "(none)" when there is none.
+func listsDevice(out string) bool {
+	listing := false
+	for _, line := range strings.Split(out, "\n") {
+		line = strings.TrimSpace(line)
+		switch {
+		case strings.HasPrefix(line, "Available devices"):
+			listing = true
+		case listing && line != "" && line != "(none)":
+			return true
+		}
+	}
+	return false
+}
+
 // packages installs the operating system packages a build needs.
 //
 // Only those missing, and from the node's own package sources: a package the
@@ -474,6 +621,11 @@ func (in *Installer) download(ctx context.Context, repo, tag, asset, dst, want s
 // untarGz unpacks an archive under dir, refusing anything that would land
 // outside it. A release archive is somebody else's file, and a path of
 // "../../etc/cron.d/x" in it would otherwise be written as root.
+//
+// Every write goes through an os.Root, which refuses to leave dir however the
+// path gets there. Checking names alone was not enough: "a -> ." followed by
+// "a/a/b -> ../.." reads as inside and resolves two levels above it, and a
+// file written through that lands wherever the links point.
 func untarGz(archive, dir string) error {
 	f, err := os.Open(archive) //nolint:gosec // inside the install root
 	if err != nil {
@@ -484,6 +636,11 @@ func untarGz(archive, dir string) error {
 	if err != nil {
 		return err
 	}
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		return err
+	}
+	defer root.Close()
 	tr := tar.NewReader(gz)
 	for {
 		h, err := tr.Next()
@@ -493,22 +650,22 @@ func untarGz(archive, dir string) error {
 		if err != nil {
 			return err
 		}
-		target, err := within(dir, h.Name)
-		if err != nil {
+		name := filepath.Clean(h.Name)
+		if _, err := within(dir, name); err != nil {
 			return err
 		}
 		switch h.Typeflag {
 		case tar.TypeDir:
-			if err := os.MkdirAll(target, 0o755); err != nil {
+			if err := root.MkdirAll(name, 0o755); err != nil {
 				return err
 			}
 		case tar.TypeReg:
-			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+			if err := root.MkdirAll(filepath.Dir(name), 0o755); err != nil {
 				return err
 			}
-			out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755) //nolint:gosec // checked above
+			out, err := root.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755)
 			if err != nil {
-				return err
+				return fmt.Errorf("%s: %w", h.Name, err)
 			}
 			if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // a release archive, size bounded by its checksum
 				out.Close()
@@ -518,19 +675,19 @@ func untarGz(archive, dir string) error {
 				return err
 			}
 		case tar.TypeSymlink:
-			// Libraries are versioned through symlinks, and those are
-			// relative. One pointing anywhere else is refused.
-			if filepath.IsAbs(h.Linkname) {
-				return fmt.Errorf("%s links outside the archive", h.Name)
+			// Libraries are versioned through symlinks to a file beside them —
+			// libllama.so -> libllama.so.0 — and that is all a release needs.
+			// A target with a separator in it could point anywhere once other
+			// links are followed, and a binary found through one would be run
+			// as root, so it is refused rather than reasoned about.
+			if h.Linkname == "" || strings.ContainsAny(h.Linkname, `/\`) || h.Linkname == ".." || h.Linkname == "." {
+				return fmt.Errorf("%s links to %q; only a file beside it is allowed", h.Name, h.Linkname)
 			}
-			if _, err := within(dir, filepath.Join(filepath.Dir(h.Name), h.Linkname)); err != nil {
-				return fmt.Errorf("%s links outside the archive", h.Name)
-			}
-			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+			if err := root.MkdirAll(filepath.Dir(name), 0o755); err != nil {
 				return err
 			}
-			_ = os.Remove(target)
-			if err := os.Symlink(h.Linkname, target); err != nil {
+			_ = root.Remove(name)
+			if err := root.Symlink(h.Linkname, name); err != nil {
 				return err
 			}
 		default:
@@ -607,6 +764,13 @@ func (in *Installer) report(st Status) {
 	}
 }
 
+func runWithStdin(ctx context.Context, env []string, stdin, name string, args ...string) ([]byte, error) {
+	cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // fixed commands and paths we installed
+	cmd.Env = append(os.Environ(), env...)
+	cmd.Stdin = strings.NewReader(stdin)
+	return cmd.CombinedOutput()
+}
+
 func (in *Installer) run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
 	if in.Run != nil {
 		return in.Run(ctx, env, name, args...)
@@ -662,5 +826,11 @@ func firstVersionLine(b []byte) string {
 			return strings.TrimSpace(l)
 		}
 	}
+	// vllm --version says the number and nothing else.
+	for _, l := range strings.Split(string(b), "\n") {
+		if l = strings.TrimSpace(l); l != "" && l[0] >= '0' && l[0] <= '9' {
+			return l
+		}
+	}
 	return ""
 }

internal/agent/provision/install_test.go

@@ -89,7 +89,11 @@ type fakeSystem struct {
 	installed map[string]bool
 	tools     map[string]bool
 	lddOut    string
-	calls     []string
+	// devices is what llama-server --list-devices prints, and backendLdd
+	// what ldd says of its GPU backend.
+	devices    string
+	backendLdd string
+	calls      []string
 	// locked is how many more package commands find the lock held.
 	locked int
 }
@@ -127,8 +131,12 @@ func (f *fakeSystem) run(_ context.Context, _ []string, name string, args ...str
 			}
 		}
 		return nil, nil
+	case name == "ldd" && strings.HasSuffix(args[0], "libggml-cuda.so"):
+		return []byte(f.backendLdd), nil
 	case name == "ldd":
 		return []byte(f.lddOut), nil
+	case strings.HasSuffix(name, "llama-server") && args[0] == "--list-devices":
+		return []byte(f.devices), nil
 	case strings.HasSuffix(name, "llama-server"):
 		return []byte("version: 10991 (abc)\n"), nil
 	}
@@ -247,13 +255,38 @@ func TestAnArchiveCannotWriteOutsideItsDirectory(t *testing.T) {
 func TestTheBuildFollowsTheHardware(t *testing.T) {
 	gpu := ubuntu
 	gpu.NVIDIADriver = "570.86.15"
-	b, err := chooseLlamaCpp("b1", gpu)
-	if err != nil || !strings.HasPrefix(b.Variant, "cuda") || len(b.Assets) != 2 {
-		t.Errorf("an NVIDIA node got %+v, %v; expected the CUDA build and its runtime", b, err)
+	got, err := llamaCppCandidates("b1", "", gpu)
+	if err != nil || got[0].Variant != "cuda-12.8" || len(got[0].Assets) != 2 || got[len(got)-1].Variant != "vulkan" {
+		t.Errorf("an NVIDIA node on driver 570 got %+v, %v; expected CUDA 12.8 with its runtime, then Vulkan", got, err)
 	}
-	if _, err := chooseLlamaCpp("b1", discovery.Host{OSID: "ubuntu", Arch: "riscv64"}); err == nil {
+	if _, err := llamaCppCandidates("b1", "", discovery.Host{OSID: "ubuntu", Arch: "riscv64"}); err == nil {
 		t.Error("a build was chosen for an architecture llama.cpp does not publish for")
 	}
+	for _, c := range []struct {
+		name string
+		host discovery.Host
+		want []string
+	}{
+		// Driver 580 and Turing: CUDA 13.3 first.
+		{"current", discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 8.0}, []string{"cuda-13.3", "cuda-12.8", "vulkan"}},
+		// A V100 on driver 550: CUDA 12.8 ships 7.0 as PTX, which needs 570.
+		{"V100, 550", discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.0}, []string{"vulkan"}},
+		// An RTX 3090 on 550: 8.6 is machine code in the CUDA 12.8 build.
+		{"3090, 550", discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 8.6}, []string{"cuda-12.8", "vulkan"}},
+		// A Pascal card: CUDA 12.8, not 13.3.
+		{"P40", discovery.Host{Arch: "amd64", NVIDIADriver: "575.51.03", MinComputeCapability: 6.1}, []string{"cuda-12.8", "vulkan"}},
+		// arm64 with a GPU: never the CPU build.
+		{"GH200", discovery.Host{Arch: "arm64", NVIDIADriver: "580.65.06", MinComputeCapability: 9.0}, []string{"cuda-13.3", "vulkan"}},
+	} {
+		got, err := llamaCppCandidates("b1", "", c.host)
+		var names []string
+		for _, b := range got {
+			names = append(names, b.Variant)
+		}
+		if err != nil || strings.Join(names, " ") != strings.Join(c.want, " ") {
+			t.Errorf("%s: %v, %v; want %v", c.name, names, err, c.want)
+		}
+	}
 
 	slow := ubuntu
 	slow.CPUFlags = nil
@@ -265,8 +298,8 @@ func TestTheBuildFollowsTheHardware(t *testing.T) {
 
 func TestAnEngineTheAgentCannotInstallSaysSo(t *testing.T) {
 	in := &Installer{Root: t.TempDir(), Host: ubuntu}
-	got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.29.0"}})
-	if got[0].State != Unsupported {
+	got := in.Converge(t.Context(), []Want{{Engine: "tgi", Version: "3.0.0"}})
+	if got[0].State != Unsupported || !strings.Contains(got[0].Detail, "install it by hand") {
 		t.Errorf("got %+v", got[0])
 	}
 }
@@ -330,3 +363,90 @@ func TestAPackageManagerBusyWithItsOwnUpdatesIsWaitedFor(t *testing.T) {
 		t.Error("the wait was not reported")
 	}
 }
+
+func TestAVersionThatIsAPathIsRefused(t *testing.T) {
+	// The version names a directory under the install root; "../" would put
+	// an install, and the RemoveAll that replaces one, outside it, as root.
+	in := &Installer{Root: t.TempDir(), Host: ubuntu}
+	for _, v := range []string{"../../etc", "..", ".hidden", "b1/../../x", "", "b1 && rm -rf /"} {
+		got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: v}})
+		if got[0].State != Unsupported {
+			t.Errorf("version %q: %+v", v, got[0])
+		}
+	}
+}
+
+func TestChainedLinksCannotCarryAFileOutOfTheInstall(t *testing.T) {
+	// Each link reads as inside the directory; together they point above it.
+	var buf bytes.Buffer
+	gz := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(gz)
+	_ = tw.WriteHeader(&tar.Header{Name: "a", Linkname: ".", Typeflag: tar.TypeSymlink})
+	_ = tw.WriteHeader(&tar.Header{Name: "a/a/b", Linkname: "../..", Typeflag: tar.TypeSymlink})
+	_ = tw.WriteHeader(&tar.Header{Name: "a/a/b/escaped", Mode: 0o644, Size: 1, Typeflag: tar.TypeReg})
+	_, _ = tw.Write([]byte("x"))
+	_ = tw.Close()
+	_ = gz.Close()
+
+	parent := t.TempDir()
+	dir := filepath.Join(parent, "install")
+	_ = os.MkdirAll(dir, 0o755)
+	archive := filepath.Join(parent, "evil.tar.gz")
+	_ = os.WriteFile(archive, buf.Bytes(), 0o644)
+
+	if err := untarGz(archive, dir); err == nil {
+		t.Error("an archive of chained links was unpacked")
+	}
+	if _, err := os.Stat(filepath.Join(parent, "escaped")); err == nil {
+		t.Fatal("a file was written outside the install directory")
+	}
+}
+
+func TestALibrarysVersionLinkIsStillUnpacked(t *testing.T) {
+	var buf bytes.Buffer
+	gz := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(gz)
+	_ = tw.WriteHeader(&tar.Header{Name: "lib/libllama.so.0", Mode: 0o755, Size: 1, Typeflag: tar.TypeReg})
+	_, _ = tw.Write([]byte("x"))
+	_ = tw.WriteHeader(&tar.Header{Name: "lib/libllama.so", Linkname: "libllama.so.0", Typeflag: tar.TypeSymlink})
+	_ = tw.Close()
+	_ = gz.Close()
+	dir := t.TempDir()
+	archive := filepath.Join(t.TempDir(), "ok.tar.gz")
+	_ = os.WriteFile(archive, buf.Bytes(), 0o644)
+	if err := untarGz(archive, dir); err != nil {
+		t.Fatal(err)
+	}
+	if target, err := os.Readlink(filepath.Join(dir, "lib", "libllama.so")); err != nil || target != "libllama.so.0" {
+		t.Errorf("link %q %v", target, err)
+	}
+}
+
+func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) {
+	// Found in the field: libggml-cuda.so in the build, cuBLAS nowhere the
+	// loader looks, and llama-server serving a GPU placement on the CPU.
+	cuda := "bin-ubuntu-cuda-12.8-x64.tar.gz"
+	rel := newRelease(map[string][]byte{
+		"llama-b1-" + cuda: tarGz(t, map[string]string{
+			"llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libggml-cuda.so": "x"}, nil),
+		"cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"libcudart.so.12": "x"}, nil),
+	})
+	gpu := ubuntu
+	gpu.NVIDIADriver = "570.86.15"
+	sys := &fakeSystem{installed: map[string]bool{"libgomp1": true},
+		devices:    "Available devices:\n  (none)\n",
+		backendLdd: "\tlibcublas.so.12 => not found\n\tlibcudart.so.12 => /x/libcudart.so.12\n"}
+	in, _ := installer(t, rel.serve(t), sys, gpu)
+
+	got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
+	if got[0].State != Failed || !strings.Contains(got[0].Detail, "sees no GPU") ||
+		!strings.Contains(got[0].Detail, "libcublas.so.12 is not installed") {
+		t.Fatalf("expected a failure naming the missing library: %+v", got[0])
+	}
+
+	// The same build seeing its GPU is installed.
+	sys.devices = "Available devices:\n  CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n"
+	if got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}); got[0].State != Ready {
+		t.Fatalf("a CUDA build that sees its GPU: %+v", got[0])
+	}
+}

internal/agent/provision/python.go

@@ -0,0 +1,271 @@
+package provision
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+)
+
+// A Python engine is a virtual environment built by uv: an interpreter uv
+// fetches, and the engine with every dependency from PyPI.
+//
+// Two things make it as trustworthy as an archive checked against its
+// publisher's checksum. The dependencies are resolved as PyPI stood a day
+// after the engine was published (--exclude-newer), so a node built next
+// month gets what one built today got, not whatever a dependency released
+// since — the usual way a working vLLM stops working. And the resolution is
+// written out with every file's sha256 from PyPI and installed with
+// --require-hashes, so a file that does not match what PyPI published is
+// refused. The resolved list is kept in the environment, as the record of
+// exactly what is installed.
+
+// pythonInstallTimeout bounds one installation: several gigabytes, on a
+// network that may be slow, but not forever.
+const pythonInstallTimeout = time.Hour
+
+// lockName is the resolved requirements, kept in the environment.
+const lockName = "llmgw-requirements.txt"
+
+func (in *Installer) installPython(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error {
+	ctx, cancel := context.WithTimeout(ctx, pythonInstallTimeout)
+	defer cancel()
+	// The version this build is, which for a GPU older than every build of
+	// the one asked for is an older one: its publication date is the one
+	// the dependencies are resolved as of.
+	w.Version = b.versionOr(w.Version)
+
+	if err := in.packages(ctx, b.Packages, st); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(in.root(), 0o755); err != nil {
+		return err
+	}
+	if b.FreeGB > 0 {
+		if free, err := in.free(in.root()); err == nil && free < float64(b.FreeGB) {
+			return fmt.Errorf("%s needs about %d GB free under %s and there are %.1f GB",
+				w.Engine, b.FreeGB, in.root(), free)
+		}
+	}
+
+	in.progress(st, "installing uv %s", UVVersion)
+	uv := in.one(ctx, Want{Engine: "uv", Version: UVVersion})
+	if uv.State != Ready {
+		return fmt.Errorf("installing uv, which installs %s: %s", w.Engine, uv.Detail)
+	}
+
+	in.progress(st, "asking PyPI when %s %s was published", r.PyPI, w.Version)
+	published, err := in.published(ctx, r.PyPI, w.Version)
+	if err != nil {
+		return err
+	}
+	asOf := published.Add(24 * time.Hour).UTC().Format(time.RFC3339)
+
+	cache := filepath.Join(in.root(), ".uv-cache")
+	env := []string{
+		"UV_CACHE_DIR=" + cache,
+		"UV_PYTHON_INSTALL_DIR=" + filepath.Join(in.root(), "python"),
+		// Only the interpreter uv fetched: the system's may be the wrong
+		// version, or change under the environment with the next upgrade.
+		"UV_PYTHON_PREFERENCE=only-managed",
+		// Nobody's uv.toml or pip.conf decides what is installed here.
+		"UV_NO_CONFIG=1",
+	}
+	tmp := dir + ".partial"
+	_ = os.RemoveAll(tmp)
+	defer os.RemoveAll(tmp) //nolint:errcheck // gone either way once renamed
+
+	in.progress(st, "creating a Python %s environment for %s", b.Python, w.Engine)
+	// Relocatable, because it is built under one name and renamed into place
+	// once it works: its scripts find their interpreter relative to
+	// themselves rather than by the path they were built at.
+	if out, err := in.run(ctx, env, uv.Binary, "venv", "--relocatable", "--python", b.Python, tmp); err != nil {
+		return fmt.Errorf("uv venv: %v: %s", err, tail(out))
+	}
+	python := filepath.Join(tmp, "bin", "python")
+
+	reqs := b.Requirements
+	if b.Wheel != nil {
+		// The engine from its GitHub release, pinned to the digest GitHub
+		// publishes for the file: the same trust as an archive, and uv
+		// refuses a download that does not match it.
+		in.progress(st, "asking GitHub for %s %s's %s…%s", b.Wheel.Repo, b.Wheel.Tag, b.Wheel.Prefix, b.Wheel.Suffix)
+		sums, err := in.checksums(ctx, b.Wheel.Repo, b.Wheel.Tag)
+		if err != nil {
+			return err
+		}
+		var asset, sum string
+		for name, s := range sums {
+			if strings.HasPrefix(name, b.Wheel.Prefix) && strings.HasSuffix(name, b.Wheel.Suffix) {
+				asset, sum = name, s
+			}
+		}
+		if asset == "" {
+			return fmt.Errorf("%s %s publishes no %s…%s", b.Wheel.Repo, b.Wheel.Tag, b.Wheel.Prefix, b.Wheel.Suffix)
+		}
+		url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + b.Wheel.Repo + "/releases/download/" +
+			b.Wheel.Tag + "/" + asset
+		reqs = append([]string{b.Wheel.Project + " @ " + url + "#sha256=" + sum}, reqs...)
+	}
+	// A second index, for the torch the engine was built on. Every package
+	// is taken from whichever of the two has the version resolved — both are
+	// the publishers' own — and each file is still pinned by its hash.
+	var index []string
+	if b.ExtraIndex != "" {
+		index = []string{"--extra-index-url", b.ExtraIndex, "--index-strategy", "unsafe-best-match"}
+	}
+
+	in.progress(st, "resolving %s against PyPI as of %s", strings.Join(reqs, " "), asOf[:10])
+	lock := filepath.Join(tmp, lockName)
+	args := append([]string{"pip", "compile", "--python", python, "--exclude-newer", asOf,
+		"--generate-hashes", "--quiet", "--output-file", lock}, index...)
+	args = append(args, "-")
+	if out, err := in.runStdin(ctx, env, strings.Join(reqs, "\n")+"\n", uv.Binary, args...); err != nil {
+		return fmt.Errorf("resolving %s: %v: %s", strings.Join(reqs, " "), err, tail(out))
+	}
+
+	in.progress(st, "downloading and installing %s and its dependencies: several gigabytes, which takes minutes",
+		strings.Join(reqs, " "))
+	install := append([]string{"pip", "install", "--python", python,
+		"--require-hashes", "--no-deps", "-r", lock}, index...)
+	if out, err := in.run(ctx, env, uv.Binary, install...); err != nil {
+		return fmt.Errorf("installing %s: %v: %s", strings.Join(reqs, " "), err, tail(out))
+	}
+
+	in.progress(st, "checking that %s runs", w.Engine)
+	if _, _, err := in.verify(ctx, tmp, b); err != nil {
+		return err
+	}
+	if b.GPUCheck && in.Host.NVIDIADriver != "" {
+		in.progress(st, "checking that %s (%s) computes on this node's GPU", w.Engine, b.Variant)
+		if err := in.gpuCheck(ctx, python, w, b); err != nil {
+			return err
+		}
+	}
+	m, _ := json.MarshalIndent(markerFile{
+		Engine: w.Engine, Version: w.Version, Variant: b.Variant,
+		Binary: filepath.Join("bin", b.Binary), Requirements: reqs,
+		ExcludeNewer: asOf, UV: UVVersion, Installed: time.Now().UTC(),
+	}, "", "  ")
+	if err := os.WriteFile(filepath.Join(tmp, marker), m, 0o644); err != nil { //nolint:gosec // not secret
+		return err
+	}
+	_ = os.RemoveAll(dir)
+	if err := os.Rename(tmp, dir); err != nil {
+		return err
+	}
+	// The cache is as large as the environment. Kept after a failure, so the
+	// next attempt does not download it all again; dropped after success.
+	_ = os.RemoveAll(cache)
+	in.log().Info("installed an engine", "engine", w.Engine, "version", w.Version,
+		"variant", b.Variant, "dir", dir, "as_of", asOf)
+	return nil
+}
+
+// published is when PyPI received a release's last file.
+func (in *Installer) published(ctx context.Context, project, version string) (time.Time, error) {
+	url := strings.TrimSuffix(in.pypiBase(), "/") + "/pypi/" + project + "/" + version + "/json"
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return time.Time{}, err
+	}
+	resp, err := in.client().Do(req)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("asking PyPI for %s %s: %w", project, version, err)
+	}
+	defer resp.Body.Close() //nolint:errcheck // read-only
+	if resp.StatusCode == http.StatusNotFound {
+		return time.Time{}, fmt.Errorf("PyPI has no %s %s", project, version)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return time.Time{}, fmt.Errorf("asking PyPI for %s %s: %s", project, version, resp.Status)
+	}
+	var rel struct {
+		URLs []struct {
+			Uploaded time.Time `json:"upload_time_iso_8601"`
+		} `json:"urls"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&rel); err != nil {
+		return time.Time{}, fmt.Errorf("reading PyPI's answer for %s %s: %w", project, version, err)
+	}
+	var last time.Time
+	for _, u := range rel.URLs {
+		if u.Uploaded.After(last) {
+			last = u.Uploaded
+		}
+	}
+	if last.IsZero() {
+		return time.Time{}, errors.New("PyPI lists no files for " + project + " " + version)
+	}
+	return last, nil
+}
+
+func (in *Installer) free(dir string) (float64, error) {
+	if in.FreeSpace != nil {
+		return in.FreeSpace(dir)
+	}
+	return FreeGB(dir)
+}
+
+func (in *Installer) pypiBase() string {
+	if in.PyPIBase != "" {
+		return in.PyPIBase
+	}
+	return "https://pypi.org"
+}
+
+// runStdin runs a command with input on its standard input.
+func (in *Installer) runStdin(ctx context.Context, env []string, stdin, name string, args ...string) ([]byte, error) {
+	if in.RunStdin != nil {
+		return in.RunStdin(ctx, env, stdin, name, args...)
+	}
+	return runWithStdin(ctx, env, stdin, name, args...)
+}
+
+// FreeGB is the space available to an unprivileged writer under dir.
+func FreeGB(dir string) (float64, error) {
+	var s syscall.Statfs_t
+	if err := syscall.Statfs(dir, &s); err != nil {
+		return 0, err
+	}
+	return float64(s.Bavail) * float64(s.Bsize) / (1 << 30), nil
+}
+
+// tail is the last few lines of a command's output: uv explains a failed
+// resolution over several, and the last alone is rarely the reason.
+func tail(b []byte) string {
+	lines := strings.Split(strings.TrimSpace(string(b)), "\n")
+	if len(lines) > 4 {
+		lines = lines[len(lines)-4:]
+	}
+	return strings.Join(lines, " / ")
+}
+
+// gpuProbe is what the environment is asked to run on the GPU: one small
+// operation, which needs the torch to have kernels for the card.
+const gpuProbe = `import torch
+assert torch.cuda.is_available(), "torch sees no CUDA device"
+x = torch.ones(64, device="cuda")
+print(float((x * 2).sum()), torch.cuda.get_device_name(0), torch.cuda.get_device_capability(0))`
+
+// gpuCheck runs gpuProbe in a freshly installed environment.
+func (in *Installer) gpuCheck(ctx context.Context, python string, w Want, b Build) error {
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
+	defer cancel()
+	out, err := in.run(ctx, nil, python, "-c", gpuProbe)
+	if err != nil {
+		hint := ""
+		if strings.Contains(string(out), "no kernel image") {
+			hint = "; this build has no kernels for this GPU, so try the other build for this node"
+		}
+		return fmt.Errorf("%s %s (%s) installed but cannot compute on this node's GPU: %s%s",
+			w.Engine, w.Version, b.Variant, tail(out), hint)
+	}
+	return nil
+}

internal/agent/provision/python_test.go

@@ -0,0 +1,312 @@
+package provision
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+)
+
+var gpuNode = discovery.Host{OSID: "ubuntu", OSVersion: "24.04", Arch: "amd64",
+	NVIDIADriver: "580.65.06", MinComputeCapability: 8.0}
+
+// fakePython is uv and the environments it builds.
+type fakePython struct {
+	fakeSystem
+	pmu      sync.Mutex
+	commands []string
+	failPip  bool
+	// noKernel makes the GPU check fail as a torch without kernels for the
+	// card does: in every build, or in noKernelIn only ("-cu13.").
+	noKernel   bool
+	noKernelIn string
+}
+
+func (f *fakePython) run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
+	line := name + " " + strings.Join(args, " ")
+	f.pmu.Lock()
+	f.commands = append(f.commands, line)
+	f.pmu.Unlock()
+	switch {
+	case strings.HasSuffix(name, "/uv") && len(args) == 1 && args[0] == "--version":
+		return []byte("uv 0.12.20\n"), nil
+	case strings.HasSuffix(name, "/uv") && args[0] == "venv":
+		return nil, os.MkdirAll(filepath.Join(args[len(args)-1], "bin"), 0o755)
+	case strings.HasSuffix(name, "/uv") && args[0] == "pip" && args[1] == "install":
+		if f.failPip {
+			return []byte("error: Hash mismatch for torch\n"), errExit
+		}
+		venv := filepath.Dir(filepath.Dir(args[slices.Index(args, "--python")+1]))
+		return nil, os.WriteFile(filepath.Join(venv, "bin", "vllm"), []byte("#!/bin/sh\n"), 0o755)
+	case strings.HasSuffix(name, "/bin/vllm") && args[0] == "--version":
+		return []byte("0.30.0\n"), nil
+	case strings.HasSuffix(name, "/bin/python") && args[0] == "-c":
+		if f.noKernel || (f.noKernelIn != "" && strings.Contains(name, f.noKernelIn)) {
+			return []byte("RuntimeError: CUDA error: no kernel image is available for execution on the device\n"), errExit
+		}
+		return []byte("128.0 Tesla V100-SXM2-32GB (7, 0)\n"), nil
+	}
+	return f.fakeSystem.run(ctx, env, name, args...)
+}
+
+func (f *fakePython) runStdin(_ context.Context, _ []string, stdin, name string, args ...string) ([]byte, error) {
+	f.pmu.Lock()
+	f.commands = append(f.commands, name+" "+strings.Join(args, " ")+" <<< "+strings.TrimSpace(stdin))
+	f.pmu.Unlock()
+	out := args[slices.Index(args, "--output-file")+1]
+	return nil, os.WriteFile(out, []byte("vllm==0.30.0 --hash=sha256:ab\n"), 0o644)
+}
+
+func pythonInstaller(t *testing.T, host discovery.Host, sys *fakePython) (*Installer, *release) {
+	uv := newRelease(map[string][]byte{
+		"uv-x86_64-unknown-linux-gnu.tar.gz": tarGz(t,
+			map[string]string{"uv-x86_64-unknown-linux-gnu/uv": "#!/bin/sh\n"}, nil),
+		// The fake GitHub serves every repository's release from this one.
+		"vllm-0.30.0+cu129-cp38-abi3-manylinux_2_28_x86_64.whl": []byte("the cu129 wheel"),
+	})
+	gh := uv.serve(t)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/pypi/vllm/0.30.0/json", func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`{"urls":[{"upload_time_iso_8601":"2026-09-22T05:22:21.647189Z"},
+			{"upload_time_iso_8601":"2026-09-22T05:23:10.039243Z"}]}`))
+	})
+	mux.HandleFunc("/pypi/vllm/0.19.1/json", func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`{"urls":[{"upload_time_iso_8601":"2026-04-02T10:00:00Z"}]}`))
+	})
+	pypi := httptest.NewServer(mux)
+	t.Cleanup(pypi.Close)
+	return &Installer{
+		Root: t.TempDir(), Host: host, Run: sys.run, RunStdin: sys.runStdin,
+		APIBase: gh.URL, DownloadBase: gh.URL, PyPIBase: pypi.URL, HTTP: gh.Client(),
+		FreeSpace: func(string) (float64, error) { return 100, nil },
+	}, uv
+}
+
+func TestVLLMIsInstalledOnAGPUNode(t *testing.T) {
+	sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
+	in, _ := pythonInstaller(t, gpuNode, sys)
+
+	got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})
+	if got[0].State != Ready {
+		t.Fatalf("not ready: %+v", got[0])
+	}
+	dir := filepath.Join(in.Root, "vllm", "0.30.0-cu13")
+	if got[0].Binary != filepath.Join(dir, "bin", "vllm") {
+		t.Errorf("binary %s, want the environment's vllm", got[0].Binary)
+	}
+	if !sys.installed["gcc"] {
+		t.Error("gcc, which Triton compiles its launcher with, was not installed")
+	}
+	all := strings.Join(sys.commands, "\n")
+	for _, want := range []string{
+		// PyPI as it stood a day after vLLM's last file, so the dependencies
+		// are the ones it was released against.
+		"--exclude-newer 2026-09-23T05:23:10Z",
+		"--generate-hashes",
+		"--require-hashes",
+		"venv --relocatable --python 3.12",
+	} {
+		if !strings.Contains(all, want) {
+			t.Errorf("no %q in:\n%s", want, all)
+		}
+	}
+	if _, err := os.Stat(filepath.Join(dir, lockName)); err != nil {
+		t.Errorf("the resolved requirements were not kept: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(in.Root, ".uv-cache")); err == nil {
+		t.Error("the download cache was left behind after a successful install")
+	}
+
+	// Checked again on the next pass, not installed again.
+	before := len(sys.commands)
+	if again := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}}); again[0].State != Ready {
+		t.Fatalf("second pass: %+v", again[0])
+	}
+	for _, c := range sys.commands[before:] {
+		if strings.Contains(c, "pip install") {
+			t.Errorf("a ready environment was installed again: %s", c)
+		}
+	}
+}
+
+func TestVLLMIsNotForEveryNode(t *testing.T) {
+	cases := []struct {
+		name  string
+		host  discovery.Host
+		state string
+		says  string
+	}{
+		{"no GPU", ubuntu, Skipped, "has none"},
+		// Too old for CUDA 13 and for its CUDA 12.9 build both.
+		{"a driver too old for either", discovery.Host{Arch: "amd64", NVIDIADriver: "470.256.02"}, Unsupported, "driver 525"},
+		// No vLLM was ever built for Pascal.
+		{"P100", discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 6.0}, Unsupported, "no vLLM is built for GPUs of compute capability 6.0"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
+			in, uv := pythonInstaller(t, c.host, sys)
+			got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})
+			if got[0].State != c.state || !strings.Contains(got[0].Detail, c.says) {
+				t.Errorf("got %s %q, want %s saying %q", got[0].State, got[0].Detail, c.state, c.says)
+			}
+			if uv.gets != 0 || len(sys.commands) != 0 {
+				t.Errorf("a node that cannot run it still started installing: %v", sys.commands)
+			}
+		})
+	}
+}
+
+func TestAPythonEngineIsRefusedWithoutTheDiskForIt(t *testing.T) {
+	sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{"gcc": true}}}
+	in, uv := pythonInstaller(t, gpuNode, sys)
+	in.FreeSpace = func(string) (float64, error) { return 12.5, nil }
+	got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})
+	if got[0].State != Failed || !strings.Contains(got[0].Detail, "12.5 GB") {
+		t.Errorf("got %s %q", got[0].State, got[0].Detail)
+	}
+	if uv.gets != 0 {
+		t.Error("it began downloading anyway")
+	}
+}
+
+func TestAFailedPythonInstallLeavesNothingThatLooksInstalled(t *testing.T) {
+	sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{"gcc": true}}, failPip: true}
+	in, _ := pythonInstaller(t, gpuNode, sys)
+	got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})
+	if got[0].State != Failed || !strings.Contains(got[0].Detail, "Hash mismatch") {
+		t.Fatalf("got %s %q", got[0].State, got[0].Detail)
+	}
+	entries, _ := os.ReadDir(filepath.Join(in.Root, "vllm"))
+	for _, e := range entries {
+		t.Errorf("left behind: %s", e.Name())
+	}
+}
+
+func TestAVersionPyPIDoesNotHaveIsSaidSo(t *testing.T) {
+	sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{"gcc": true}}}
+	in, _ := pythonInstaller(t, gpuNode, sys)
+	got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "9.9.9"}})
+	if got[0].State != Failed || !strings.Contains(got[0].Detail, "PyPI has no vllm 9.9.9") {
+		t.Errorf("got %s %q", got[0].State, got[0].Detail)
+	}
+}
+
+func TestAnOlderDriverGetsTheSameVLLMBuiltForCUDA12(t *testing.T) {
+	// A Turing card on a driver too old for CUDA 13: the same vLLM's CUDA
+	// 12.9 build, from the project's GitHub release, with the torch PyTorch
+	// publishes for 12.9.
+	volta := discovery.Host{OSID: "ubuntu", Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.5}
+	sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
+	in, gh := pythonInstaller(t, volta, sys)
+	got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})
+	if got[0].State != Ready || got[0].Variant != "cu129" {
+		t.Fatalf("a Turing node on driver 550: %+v", got[0])
+	}
+	if _, err := os.Stat(filepath.Join(in.Root, "vllm", "0.30.0-cu129", marker)); err != nil {
+		t.Errorf("not installed under its own build's name: %v", err)
+	}
+	all := strings.Join(sys.commands, "\n")
+	wheel := "vllm-0.30.0+cu129-cp38-abi3-manylinux_2_28_x86_64.whl"
+	for _, want := range []string{
+		// Pinned to the digest GitHub publishes for the wheel.
+		"vllm @ " + in.DownloadBase + "/vllm-project/vllm/releases/download/v0.30.0/" + wheel + "#sha256=" +
+			strings.TrimPrefix(gh.digests[wheel], "sha256:"),
+		"--extra-index-url https://download.pytorch.org/whl/cu129 --index-strategy unsafe-best-match -",
+		"--require-hashes --no-deps -r",
+		// Computing on the card, not only importing.
+		"/bin/python -c import torch",
+	} {
+		if !strings.Contains(all, want) {
+			t.Errorf("no %q in:\n%s", want, all)
+		}
+	}
+	for _, c := range sys.commands {
+		if strings.Contains(c, "pip install") && !strings.Contains(c, "download.pytorch.org/whl/cu129") {
+			t.Errorf("installed without the index its torch comes from: %s", c)
+		}
+	}
+}
+
+func TestABuildChosenForANodeIsTheOneInstalled(t *testing.T) {
+	sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
+	in, _ := pythonInstaller(t, gpuNode, sys)
+	if got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0", Variant: "cu129"}}); got[0].Variant != "cu129" || got[0].State != Ready {
+		t.Errorf("cu129 chosen for a node that runs cu13 too: %+v", got[0])
+	}
+	if _, err := vllmCandidates("0.30.0", "cu200", gpuNode); err == nil {
+		t.Error("a build that does not exist was accepted")
+	}
+}
+
+func TestABuildThatCannotComputeOnTheCardIsNotInstalled(t *testing.T) {
+	sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}, noKernel: true}
+	in, _ := pythonInstaller(t, gpuNode, sys)
+	got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})
+	if got[0].State != Failed || !strings.Contains(got[0].Detail, "no kernels for this GPU") {
+		t.Fatalf("a build without kernels for the card: %+v", got[0])
+	}
+	if _, err := os.Stat(filepath.Join(in.Root, "vllm", "0.30.0-cu13", marker)); err == nil {
+		t.Error("marked installed")
+	}
+}
+
+func TestOnlyACUDARequirementFallsBackToTheCUDA12Build(t *testing.T) {
+	// A machine vLLM is not published for is refused, not given the other
+	// build to fail with.
+	if _, err := vllmCandidates("0.30.0", "", discovery.Host{NVIDIADriver: "580.65.06"}); err == nil ||
+		!strings.Contains(err.Error(), "no Linux build") {
+		t.Errorf("an unknown architecture: %v", err)
+	}
+	if _, err := vllmCandidates("0.30.0", "cu129", ubuntu); !errors.As(err, new(NotHere)) {
+		t.Errorf("a node with no GPU chosen cu129: %v", err)
+	}
+	// Driver 580 and Turing: CUDA 13 first, the CUDA 12.9 build behind it.
+	got, err := vllmCandidates("0.30.0", "", gpuNode)
+	if err != nil || len(got) != 2 || got[0].Variant != "cu13" || got[1].Variant != "cu129" {
+		t.Errorf("candidates on a current node: %+v, %v", got, err)
+	}
+}
+
+func TestAV100GetsTheNewestVLLMBuiltForItByItself(t *testing.T) {
+	// 0.30.0 asked for; no build of it has Volta kernels. The node installs
+	// 0.19.1, the newest that has, and says so.
+	v100 := discovery.Host{OSID: "ubuntu", Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.0}
+	sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
+	in, _ := pythonInstaller(t, v100, sys)
+	got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})[0]
+	if got.State != Ready || got.Version != "0.19.1" || got.Variant != "cu12" ||
+		!strings.Contains(got.Detail, "the newest built for compute capability 7.0, instead of 0.30.0") {
+		t.Fatalf("a V100: %+v", got)
+	}
+	if _, err := os.Stat(filepath.Join(in.Root, "vllm", "0.19.1-cu12", marker)); err != nil {
+		t.Errorf("not installed as what it is: %v", err)
+	}
+	all := strings.Join(sys.commands, "\n")
+	if !strings.Contains(all, "<<< vllm==0.19.1") || strings.Contains(all, "download.pytorch.org") {
+		t.Errorf("0.19.1 is PyPI's CUDA 12 build, with PyPI's torch:\n%s", all)
+	}
+	// Chosen cu13, which 0.19.1 has none of: refused, saying why.
+	if _, err := vllmCandidates("0.30.0", "cu13", v100); err == nil {
+		t.Error("a CUDA 13 build was offered to a V100")
+	}
+}
+
+func TestABuildThatFailsOnTheCardGivesWayToTheNext(t *testing.T) {
+	// The table says CUDA 13 runs here; the card says otherwise. The next
+	// build is installed rather than the node going without.
+	sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}, noKernelIn: "-cu13."}
+	in, _ := pythonInstaller(t, gpuNode, sys)
+	got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})[0]
+	if got.State != Ready || got.Variant != "cu129" {
+		t.Fatalf("after the CUDA 13 build failed on the card: %+v", got)
+	}
+}

internal/agent/provision/recipe.go

@@ -11,6 +11,7 @@ package provision
 
 import (
 	"fmt"
+	"slices"
 	"strings"
 
 	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
@@ -33,36 +34,416 @@ type Build struct {
 	// CPUFlags are extensions the build is slow without. Missing ones are
 	// reported, not refused: slow is still running.
 	CPUFlags []string
+
+	// Python, when set, makes the build a Python environment rather than an
+	// archive: that interpreter version, with Requirements installed from
+	// PyPI by uv. Binary is then looked for in the environment's bin.
+	Python       string
+	Requirements []string
+	// VersionArgs is what Binary is run with to prove the build works, when
+	// --version is not it.
+	VersionArgs []string
+	// GPUBackend, when set, is the shared library the build computes on GPUs
+	// with, loaded at run time rather than linked: "libggml-cuda.so". The
+	// build must list a device with it, or it is not installed — a llama.cpp
+	// whose CUDA backend cannot load runs on the CPU without complaint.
+	GPUBackend string
+	// Wheel, when set, is the engine itself as a wheel from a GitHub
+	// release rather than from PyPI, checked against the digest GitHub
+	// publishes for it. ExtraIndex is a package index besides PyPI that the
+	// rest resolves from too: the CUDA 12.9 torch the wheel was built on.
+	Wheel      *Wheel
+	ExtraIndex string
+	// GPUCheck, on a node with an NVIDIA GPU, has the installed environment
+	// compute something on one before the build counts as installed. A build
+	// with no kernels for the card — a GPU generation its torch dropped —
+	// imports and prints its version, and fails on its first model.
+	GPUCheck bool
+	// Version is the engine version this build is, when it is not the one
+	// asked for: the newest built for this machine's GPUs, where the one
+	// asked for has no kernels for them. Empty is the version asked for.
+	Version string
+	// Note says why this build, when it is not the plain one: shown beside
+	// it wherever the installation is.
+	Note string
+	// FreeGB is the disk space the installation needs before it starts: a
+	// Python engine is several gigabytes, and one that fills the disk halfway
+	// leaves the node worse off than one that never began.
+	FreeGB int
+}
+
+// Wheel is a package published as a GitHub release asset: the asset whose
+// name starts with Prefix and ends with Suffix.
+type Wheel struct {
+	Project, Repo, Tag, Prefix, Suffix string
 }
 
 // Recipe knows how to install one engine.
 type Recipe struct {
-	// Repo is the GitHub repository whose releases carry the builds.
+	// Repo is the GitHub repository whose releases carry the builds; empty
+	// for an engine installed from PyPI.
 	Repo string
-	// Choose picks the build for a machine, or says why none fits.
-	Choose func(version string, h discovery.Host) (Build, error)
+	// PyPI is the project an engine installed from PyPI is published as.
+	PyPI string
+	// Candidates are the builds that may run on a machine, best first, or
+	// why none can. The installer takes the first that installs and is seen
+	// to work — computing on the GPU, where there is one — so a build this
+	// table is wrong about costs a download, not a node. variant is a build
+	// chosen for the node; empty leaves the choice to the machine.
+	Candidates func(version, variant string, h discovery.Host) ([]Build, error)
 }
 
 // Recipes are the engines this agent can install.
 //
-// vLLM and SGLang are not here yet: they are Python environments of several
-// gigabytes rather than an archive, and are installed by hand and named in the
-// agent's configuration until they are.
+// vLLM and SGLang are Python environments of several gigabytes, installed
+// from PyPI by uv, which is itself installed like llama.cpp: a GitHub release
+// checked against its published checksum.
 var Recipes = map[string]Recipe{
-	"llamacpp": {Repo: "ggml-org/llama.cpp", Choose: chooseLlamaCpp},
+	"llamacpp": {Repo: "ggml-org/llama.cpp", Candidates: llamaCppCandidates},
+	"vllm":     {PyPI: "vllm", Candidates: vllmCandidates},
+	"sglang":   {PyPI: "sglang", Candidates: sglangCandidates},
+	"uv":       {Repo: "astral-sh/uv", Candidates: uvCandidates},
 }
 
-// cudaForLlamaCpp is the CUDA build taken on an NVIDIA node. The release also
-// carries a matching cudart archive, so the node needs a driver and nothing
-// else — no toolkit.
-const cudaForLlamaCpp = "12.8"
+// Installable are the engines this agent can install, as it tells the
+// control plane: every recipe but uv, which is a tool for the others.
+func Installable() []string {
+	var out []string
+	for name := range Recipes {
+		if name != "uv" {
+			out = append(out, name)
+		}
+	}
+	slices.Sort(out)
+	return out
+}
 
-func chooseLlamaCpp(version string, h discovery.Host) (Build, error) {
-	// The published Linux builds are built on Ubuntu, but nothing about them
-	// is Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which
-	// Debian 12 and RHEL 10 have and RHEL 9 does not. So the distribution's
-	// name decides nothing here. The build is tried everywhere, and ldd says
-	// whether this system can load it — in the words of the missing symbol.
+// UVVersion is the uv that installs the Python engines.
+const UVVersion = "0.12.20"
+
+// pythonForEngines is the interpreter the Python engines run on. uv fetches
+// it, so the node's own Python — or its absence — does not matter.
+const pythonForEngines = "3.12"
+
+// NotHere is the error a recipe gives for a node the engine is not meant
+// for, as opposed to one it should run on and cannot: vLLM on a machine
+// with no GPU is not a fault, and is not reported as one.
+type NotHere struct{ Reason string }
+
+func (e NotHere) Error() string { return e.Reason }
+
+func uvCandidates(version, _ string, h discovery.Host) ([]Build, error) {
+	b, err := chooseUV(version, h)
+	if err != nil {
+		return nil, err
+	}
+	return []Build{b}, nil
+}
+
+func chooseUV(_ string, h discovery.Host) (Build, error) {
+	var arch string
+	switch h.Arch {
+	case "amd64":
+		arch = "x86_64"
+	case "arm64":
+		arch = "aarch64"
+	default:
+		return Build{}, fmt.Errorf("uv publishes no Linux build for %s", orUnknown(h.Arch))
+	}
+	return Build{
+		Variant: arch,
+		Assets:  []string{"uv-" + arch + "-unknown-linux-gnu.tar.gz"},
+		Binary:  "uv",
+	}, nil
+}
+
+// What builds need of the NVIDIA side of a machine, read from what each
+// project publishes and how it was built:
+//
+//   - CUDA 13 needs driver 580; CUDA 12 builds run on driver 525 and newer,
+//     by CUDA's minor version compatibility — except code shipped only as
+//     PTX, which the driver compiles and so needs a driver as new as the
+//     CUDA it was compiled with.
+//   - Which GPUs a build has kernels for is the build's own list, not what
+//     its CUDA supports.
+const (
+	minDriverForCUDA13 = 580
+	minDriverForCUDA12 = 525
+)
+
+// gpuNeeds is what one build needs of a machine's GPUs and driver.
+type gpuNeeds struct {
+	driver int
+	cc     float64
+}
+
+// fits says why a machine cannot run a build, or nil.
+func (n gpuNeeds) fits(what string, h discovery.Host) error {
+	if major := leadingInt(h.NVIDIADriver); n.driver > 0 && major > 0 && major < n.driver {
+		return fmt.Errorf("%s needs NVIDIA driver %d or newer; this node has %s", what, n.driver, h.NVIDIADriver)
+	}
+	if cc := h.MinComputeCapability; n.cc > 0 && cc > 0 && cc < n.cc {
+		return fmt.Errorf("%s has kernels for compute capability %.1f and newer; this node has %.1f", what, n.cc, cc)
+	}
+	return nil
+}
+
+// candidate is a build and what it needs.
+type candidate struct {
+	b     Build
+	needs gpuNeeds
+	what  string
+}
+
+// pick keeps the candidates a machine can run, in order, and those a
+// variant chosen for the node names; with none left, it says why each went.
+func pick(engine, variant string, cands []candidate, h discovery.Host) ([]Build, error) {
+	var out []Build
+	var why []string
+	for _, c := range cands {
+		if variant != "" && c.b.Variant != variant {
+			continue
+		}
+		if err := c.needs.fits(c.what, h); err != nil {
+			why = append(why, err.Error())
+			continue
+		}
+		out = append(out, c.b)
+	}
+	switch {
+	case len(out) > 0:
+		return out, nil
+	case len(why) == 0 && variant != "":
+		var names []string
+		for _, c := range cands {
+			names = append(names, c.b.Variant)
+		}
+		return nil, fmt.Errorf("%s has no build %q here; it has %s", engine, variant, strings.Join(names, ", "))
+	}
+	return nil, fmt.Errorf("no build of %s runs on this node: %s", engine, strings.Join(why, "; "))
+}
+
+// --- vLLM ---
+
+// VLLMVariants are the builds of vLLM a node can be given by choice: CUDA
+// 13 from PyPI, and CUDA 12.9 from the project's GitHub release — the same
+// version, built for drivers CUDA 13 does not run on.
+var VLLMVariants = []string{"cu13", "cu129"}
+
+// pytorchCU129 is where the CUDA 12.9 build's torch is published. PyPI's
+// torch is built for CUDA 13; the cu129 vLLM wheel needs this one.
+const pytorchCU129 = "https://download.pytorch.org/whl/cu129"
+
+// vllmCUDA13Since is the first vLLM whose PyPI build is CUDA 13, read from
+// its releases: 0.19.1 publishes "+cu130" beside a CUDA 12 default, 0.20.0
+// "+cu129" beside a CUDA 13 one.
+const vllmCUDA13Since = "0.20"
+
+// vllmMinCompute is the oldest GPU a vLLM version's builds have kernels for,
+// from its release pipeline: on x86, 7.0 (Volta) up to 0.19.1 — the
+// Dockerfile's default "7.0 7.5 8.0 8.9 9.0 10.0 12.0" — and 7.5 from 0.20.0,
+// CUDA 12.9 build included; on arm64, built for Grace machines, 8.7 before
+// 0.20 and 8.0 since.
+func vllmMinCompute(version, arch string) float64 {
+	old := versionBefore(version, vllmCUDA13Since)
+	switch {
+	case arch == "arm64" && old:
+		return 8.7
+	case arch == "arm64":
+		return 8.0
+	case old:
+		return 7.0
+	}
+	return 7.5
+}
+
+// vllmNewestFor is the newest vLLM built for GPUs of compute capability cc
+// on an architecture, for GPUs older than every build of the version asked
+// for: a V100 gets 0.19.1, where 0.20 and later have nothing for it. Nothing
+// for a GPU no vLLM was built for.
+var vllmNewestFor = []struct {
+	arch    string
+	from    float64 // the oldest GPU it covers
+	version string
+}{
+	{"amd64", 7.0, "0.19.1"},
+}
+
+// vllmCandidates are the vLLM builds for a machine.
+//
+// The version asked for, in the build the machine runs: CUDA 13 where the
+// driver allows, the same version's CUDA 12.9 build where it does not. One
+// version across the cluster wherever the GPUs allow it, because the flags
+// llmgw builds are checked against it. Where they do not — GPUs older than
+// every build of that version — the newest version built for them, said so
+// beside the installation: vLLM on a V100 at all, rather than not.
+func vllmCandidates(version, variant string, h discovery.Host) ([]Build, error) {
+	arch, err := gpuArch("vLLM", h)
+	if err != nil {
+		return nil, err
+	}
+	v, note := version, ""
+	if cc := h.MinComputeCapability; cc > 0 && cc < vllmMinCompute(version, h.Arch) {
+		older := ""
+		for _, n := range vllmNewestFor {
+			if n.arch == h.Arch && cc >= n.from {
+				older = n.version
+			}
+		}
+		if older == "" {
+			return nil, fmt.Errorf("no vLLM is built for GPUs of compute capability %.1f on %s; "+
+				"the node serves llama.cpp models", cc, arch)
+		}
+		v, note = older, fmt.Sprintf("vLLM %s, the newest built for compute capability %.1f, instead of %s", older, cc, version)
+	}
+	pypi := func(variant string) Build {
+		b := Build{
+			Variant:      variant,
+			Python:       pythonForEngines,
+			Requirements: []string{"vllm==" + v},
+			Binary:       "vllm",
+			Packages:     compilerPackages,
+			FreeGB:       25,
+			GPUCheck:     true,
+			Note:         note,
+		}
+		if v != version {
+			b.Version = v
+		}
+		return b
+	}
+	minCC := vllmMinCompute(v, h.Arch)
+	if versionBefore(v, vllmCUDA13Since) {
+		// PyPI's own build is CUDA 12, and is the one there is; asked for
+		// cu129, this is it.
+		if variant == "cu129" {
+			variant = "cu12"
+		}
+		return pick("vLLM "+v, variant, []candidate{
+			{pypi("cu12"), gpuNeeds{minDriverForCUDA12, minCC}, "vLLM " + v},
+		}, h)
+	}
+	cu129 := pypi("cu129")
+	cu129.Requirements = nil
+	// Found in the release's assets rather than named: the manylinux tag
+	// in the file's name changes between versions (2_28, 2_31, 2_34).
+	cu129.Wheel = &Wheel{
+		Project: "vllm", Repo: "vllm-project/vllm", Tag: "v" + v,
+		Prefix: "vllm-" + v + "+cu129-", Suffix: "_" + map[string]string{"amd64": "x86_64", "arm64": "aarch64"}[h.Arch] + ".whl",
+	}
+	cu129.ExtraIndex = pytorchCU129
+	return pick("vLLM "+v, variant, []candidate{
+		{pypi("cu13"), gpuNeeds{minDriverForCUDA13, minCC}, "vLLM " + v + " built for CUDA 13"},
+		{cu129, gpuNeeds{minDriverForCUDA12, minCC}, "vLLM " + v + " built for CUDA 12.9"},
+	}, h)
+}
+
+// gpuArch says whether a Python GPU engine is published for this machine
+// and whether it has an NVIDIA GPU at all.
+func gpuArch(engine string, h discovery.Host) (string, error) {
+	switch h.Arch {
+	case "amd64", "arm64":
+	default:
+		return "", fmt.Errorf("%s publishes no Linux build for %s", engine, orUnknown(h.Arch))
+	}
+	if h.NVIDIADriver == "" {
+		return "", NotHere{engine + " runs on NVIDIA GPUs and this node has none; it serves llama.cpp models"}
+	}
+	return h.Arch, nil
+}
+
+// --- SGLang ---
+
+// sglangCUDA13Since is the first SGLang built for CUDA 13.
+const sglangCUDA13Since = "0.5"
+
+func sglangCandidates(version, variant string, h discovery.Host) ([]Build, error) {
+	if _, err := gpuArch("SGLang", h); err != nil {
+		return nil, err
+	}
+	b := Build{
+		Variant:      "cu13",
+		Python:       pythonForEngines,
+		Requirements: []string{"sglang==" + version},
+		// SGLang is launched as a module, so what the agent is given is the
+		// environment's interpreter, and the proof is importing it.
+		Binary:      "python",
+		VersionArgs: []string{"-c", "import sglang; print('sglang', sglang.__version__)"},
+		Packages:    compilerPackages,
+		FreeGB:      25,
+		GPUCheck:    true,
+	}
+	needs := gpuNeeds{minDriverForCUDA13, 7.5}
+	if versionBefore(version, sglangCUDA13Since) {
+		b.Variant, needs = "cu12", gpuNeeds{minDriverForCUDA12, 7.0}
+	}
+	return pick("SGLang "+version, variant, []candidate{{b, needs, "SGLang " + version}}, h)
+}
+
+// Python engines need a C compiler at run time: Triton builds a small
+// launcher for its kernels the first time a model loads, and without gcc
+// that load fails long after the installation looked finished.
+var compilerPackages = map[string][]string{"deb": {"gcc"}, "rpm": {"gcc"}}
+
+// versionBefore reports whether version a is older than b, comparing the
+// dotted numbers in turn ("0.25.1" before "0.26"). A part that is not a
+// number counts as zero.
+func versionBefore(a, b string) bool {
+	if b == "" {
+		return false
+	}
+	pa, pb := strings.Split(strings.TrimPrefix(a, "v"), "."), strings.Split(b, ".")
+	for i := 0; i < max(len(pa), len(pb)); i++ {
+		x, y := 0, 0
+		if i < len(pa) {
+			x = leadingInt(pa[i])
+		}
+		if i < len(pb) {
+			y = leadingInt(pb[i])
+		}
+		if x != y {
+			return x < y
+		}
+	}
+	return false
+}
+
+// leadingInt is the number a version string starts with: 580 of "580.65.06".
+func leadingInt(s string) int {
+	n := 0
+	for _, c := range strings.TrimSpace(s) {
+		if c < '0' || c > '9' {
+			break
+		}
+		n = n*10 + int(c-'0')
+	}
+	return n
+}
+
+// --- llama.cpp ---
+
+// llamaCUDA12Real are the GPUs llama.cpp's CUDA 12.8 build carries machine
+// code for. The rest of what it covers — 5.0, 6.1, 7.0, 7.5, 8.0, 9.0 — it
+// carries as PTX, which a driver older than the CUDA it came from cannot
+// compile.
+var llamaCUDA12Real = []float64{8.6, 8.9, 12.0}
+
+// llamaCppCandidates are the llama.cpp builds for a machine.
+//
+// On an NVIDIA node, the GPU builds best first: CUDA 13.3 (driver 580,
+// Turing and newer), CUDA 12.8 on x64 (Maxwell to Blackwell; driver 570 for
+// the GPUs it ships as PTX, 525 for those it ships as machine code), and
+// Vulkan, which runs on any card whose driver has Vulkan — the generations
+// CUDA builds no longer cover. Never the CPU build on an NVIDIA node: that is
+// a GPU placement served on the CPU, which is what installing checks for.
+// The CPU build where there is no NVIDIA GPU.
+//
+// The published Linux builds are built on Ubuntu, but nothing about them is
+// Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which Debian 12
+// and RHEL 10 have and RHEL 9 does not. So the distribution's name decides
+// nothing here. The build is tried everywhere, and ldd says whether this
+// system can load it — in the words of the missing symbol.
+func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, error) {
 	var arch string
 	switch h.Arch {
 	case "amd64":
@@ -70,35 +451,53 @@ func chooseLlamaCpp(version string, h discovery.Host) (Build, error) {
 	case "arm64":
 		arch = "arm64"
 	default:
-		return Build{}, fmt.Errorf("llama.cpp publishes no Linux build for %s", orUnknown(h.Arch))
+		return nil, fmt.Errorf("llama.cpp publishes no Linux build for %s", orUnknown(h.Arch))
 	}
-
-	b := Build{
-		Variant: "cpu",
-		Assets:  []string{fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)},
-		// Everything else the build links is in the archive or in any current
-		// glibc system; OpenMP's runtime is the one a cloud image lacks, and
-		// the server does not start without it.
-		Packages: map[string][]string{"deb": {"libgomp1"}, "rpm": {"libgomp"}},
-		Binary:   "llama-server",
-	}
-	if arch == "x64" {
-		// The archive carries a backend per CPU generation and loads the best
-		// one, down to SSE4.2. Below AVX2 it runs, several times slower.
-		b.CPUFlags = []string{"avx2"}
-	}
-	// The CUDA build is published for x64 at this version; on arm64 it is a
-	// newer CUDA that the driver may not support, so an arm64 GPU node runs
-	// the CPU build until someone decides otherwise.
-	if h.NVIDIADriver != "" && arch == "x64" {
-		name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp)
-		b.Variant = "cuda-" + cudaForLlamaCpp
-		b.Assets = []string{
-			fmt.Sprintf("llama-%s-%s", version, name),
-			fmt.Sprintf("cudart-llama-%s-%s", version, name),
+	base := func(variant string, assets ...string) Build {
+		b := Build{
+			Variant: variant,
+			Assets:  assets,
+			// Everything else the build links is in the archive or in any
+			// current glibc system; OpenMP's runtime is the one a cloud image
+			// lacks, and the server does not start without it.
+			Packages: map[string][]string{"deb": {"libgomp1"}, "rpm": {"libgomp"}},
+			Binary:   "llama-server",
 		}
+		if arch == "x64" {
+			// The archive carries a backend per CPU generation and loads the
+			// best one, down to SSE4.2. Below AVX2 it runs, several times
+			// slower.
+			b.CPUFlags = []string{"avx2"}
+		}
+		return b
 	}
-	return b, nil
+	if h.NVIDIADriver == "" {
+		return pick("llama.cpp", variant, []candidate{
+			{base("cpu", fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)), gpuNeeds{}, "llama.cpp"},
+		}, h)
+	}
+	cuda := func(cv string) Build {
+		name := fmt.Sprintf("bin-ubuntu-cuda-%s-%s.tar.gz", cv, arch)
+		// The release carries the CUDA runtime beside the build, so the node
+		// needs a driver and nothing else — no toolkit.
+		b := base("cuda-"+cv, fmt.Sprintf("llama-%s-%s", version, name), fmt.Sprintf("cudart-llama-%s-%s", version, name))
+		b.GPUBackend = "libggml-cuda.so"
+		return b
+	}
+	vulkan := base("vulkan", fmt.Sprintf("llama-%s-bin-ubuntu-vulkan-%s.tar.gz", version, arch))
+	vulkan.GPUBackend = "libggml-vulkan.so"
+	vulkan.Packages = map[string][]string{"deb": {"libgomp1", "libvulkan1"}, "rpm": {"libgomp", "vulkan-loader"}}
+
+	cuda12Driver := 570
+	if slices.Contains(llamaCUDA12Real, h.MinComputeCapability) {
+		cuda12Driver = minDriverForCUDA12
+	}
+	cands := []candidate{{cuda("13.3"), gpuNeeds{minDriverForCUDA13, 7.5}, "llama.cpp built for CUDA 13.3"}}
+	if arch == "x64" {
+		cands = append(cands, candidate{cuda("12.8"), gpuNeeds{cuda12Driver, 5.0}, "llama.cpp built for CUDA 12.8"})
+	}
+	cands = append(cands, candidate{vulkan, gpuNeeds{}, "llama.cpp built for Vulkan"})
+	return pick("llama.cpp", variant, cands, h)
 }
 
 func orUnknown(s string) string {

internal/agent/supervisor/health.go

@@ -53,8 +53,8 @@ func NewHTTPHealth(timeout time.Duration, opts ...HTTPHealthOption) *HTTPHealth
 }
 
 // Ready reports whether the engine at baseURL is serving model.
-func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
-	if err := h.get(ctx, baseURL+h.healthPath, nil); err != nil {
+func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model, apiKey string) error {
+	if err := h.get(ctx, baseURL+h.healthPath, apiKey, nil); err != nil {
 		return fmt.Errorf("health: %w", err)
 	}
 	if !h.requireName || model == "" {
@@ -66,7 +66,7 @@ func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
 			ID string `json:"id"`
 		} `json:"data"`
 	}
-	if err := h.get(ctx, baseURL+h.modelsPath, &list); err != nil {
+	if err := h.get(ctx, baseURL+h.modelsPath, apiKey, &list); err != nil {
 		return fmt.Errorf("models: %w", err)
 	}
 	for _, d := range list.Data {
@@ -77,11 +77,14 @@ func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
 	return fmt.Errorf("engine is up but does not serve %q yet", model)
 }
 
-func (h *HTTPHealth) get(ctx context.Context, url string, out any) error {
+func (h *HTTPHealth) get(ctx context.Context, url, apiKey string, out any) error {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
 		return err
 	}
+	if apiKey != "" {
+		req.Header.Set("Authorization", "Bearer "+apiKey)
+	}
 	resp, err := h.client.Do(req)
 	if err != nil {
 		return err

internal/agent/supervisor/health_test.go

@@ -39,7 +39,7 @@ func engineStub(t *testing.T, healthy bool, models ...string) *httptest.Server {
 
 func TestReadyWhenServingModel(t *testing.T) {
 	srv := engineStub(t, true, "llama-3.3-70b")
-	if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b"); err != nil {
+	if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", ""); err != nil {
 		t.Errorf("a serving engine was reported unready: %v", err)
 	}
 }
@@ -48,7 +48,7 @@ func TestNotReadyWhileLoading(t *testing.T) {
 	// The case that matters: the process is up but the weights are not loaded.
 	// Routing on /health alone would send traffic into a loading model.
 	srv := engineStub(t, false, "llama-3.3-70b")
-	err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b")
+	err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", "")
 	if err == nil {
 		t.Fatal("an unhealthy engine was reported ready")
 	}
@@ -61,7 +61,7 @@ func TestNotReadyWhenModelMissing(t *testing.T) {
 	// /health passes but the model is not listed: the engine is up and serving
 	// something else, or still registering.
 	srv := engineStub(t, true, "some-other-model")
-	err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b")
+	err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", "")
 	if err == nil {
 		t.Fatal("a model that is not served was reported ready")
 	}
@@ -73,14 +73,33 @@ func TestNotReadyWhenModelMissing(t *testing.T) {
 func TestModelCheckCanBeSkipped(t *testing.T) {
 	srv := engineStub(t, true)
 	h := NewHTTPHealth(time.Second, WithoutModelCheck())
-	if err := h.Ready(t.Context(), srv.URL, "anything"); err != nil {
+	if err := h.Ready(t.Context(), srv.URL, "anything", ""); err != nil {
 		t.Errorf("the model check was not skipped: %v", err)
 	}
 }
 
 func TestUnreachableEngine(t *testing.T) {
 	h := NewHTTPHealth(200 * time.Millisecond)
-	if err := h.Ready(t.Context(), "http://127.0.0.1:1", "m"); err == nil {
+	if err := h.Ready(t.Context(), "http://127.0.0.1:1", "m", ""); err == nil {
 		t.Error("an unreachable engine was reported ready")
 	}
 }
+
+func TestTheHealthCheckSendsTheEnginesKey(t *testing.T) {
+	// An engine started with a key refuses /v1/models without it, and a
+	// shard that is serving would read as never ready.
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer k1" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		_, _ = w.Write([]byte(`{"data":[{"id":"m"}]}`))
+	}))
+	defer srv.Close()
+	if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "m", "k1"); err != nil {
+		t.Errorf("with the key: %v", err)
+	}
+	if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "m", ""); err == nil {
+		t.Error("without the key the engine refused, and that read as ready")
+	}
+}

internal/agent/supervisor/supervisor.go

@@ -79,7 +79,7 @@ func (c *Config) normalize() {
 // supervisor can be tested without an HTTP engine.
 type HealthChecker interface {
 	// Ready reports whether the shard at baseURL serves model.
-	Ready(ctx context.Context, baseURL, model string) error
+	Ready(ctx context.Context, baseURL, model, apiKey string) error
 }
 
 // Shard is one supervised engine process.
@@ -243,6 +243,16 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
 	}
 
 	cmd := cmds[len(cmds)-1]
+	// The main process can need the rendezvous point too: a Ray worker's
+	// join is its only process, and started before the head listens it
+	// exits, failing the shard and with it the replica.
+	if cmd.WaitForLeaderPort && !sh.Spec.Leader() {
+		addr := fmt.Sprintf("%s:%d", sh.Spec.LeaderAddr, sh.Spec.LeaderPort)
+		if err := waitForPort(runCtx, addr, s.cfg.RendezvousTimeout); err != nil {
+			cancel()
+			return fmt.Errorf("waiting for %s (%s): %w", addr, cmd.Name, err)
+		}
+	}
 
 	c := exec.CommandContext(runCtx, cmd.Path, cmd.Args...)
 	c.Env = cmd.Env
@@ -401,7 +411,7 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
 			last := strings.Join(sh.Logs(10), "\n")
 			return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last)
 		case <-t.C:
-			if err := s.health.Ready(ctx, baseURL, model); err != nil {
+			if err := s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); err != nil {
 				continue
 			}
 			sh.setState(Ready, nil)

internal/agent/supervisor/supervisor_test.go

@@ -21,7 +21,7 @@ import (
 // scriptedHealth reports ready once the flag is set.
 type scriptedHealth struct{ ready atomic.Bool }
 
-func (h *scriptedHealth) Ready(context.Context, string, string) error {
+func (h *scriptedHealth) Ready(context.Context, string, string, string) error {
 	if h.ready.Load() {
 		return nil
 	}
@@ -243,7 +243,9 @@ func TestWaitReadyTimeoutIncludesLogs(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	time.Sleep(100 * time.Millisecond) // let the line be captured
+	eventually(t, 3*time.Second, "the line was never captured", func() bool {
+		return strings.Contains(strings.Join(sh.Logs(10), "\n"), "CUDA out of memory")
+	})
 
 	err = s.WaitReady(t.Context(), sh, "http://x", "m", 200*time.Millisecond)
 	if err == nil {
@@ -295,7 +297,10 @@ func TestLogsAreCaptured(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	time.Sleep(400 * time.Millisecond)
+	eventually(t, 3*time.Second, "not every line arrived", func() bool {
+		l := strings.Join(sh.Logs(10), "\n")
+		return strings.Contains(l, "line1") && strings.Contains(l, "line2") && strings.Contains(l, "line3")
+	})
 
 	logs := strings.Join(sh.Logs(10), "\n")
 	for _, want := range []string{"line1", "line2", "line3"} {
@@ -372,11 +377,11 @@ func TestStateFileRemovedOnExit(t *testing.T) {
 
 	sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 0"))
 	waitState(t, sh, Failed, 3*time.Second)
-	time.Sleep(50 * time.Millisecond)
-
-	if _, err := os.Stat(s.statePath("a")); !os.IsNotExist(err) {
-		t.Error("the state file outlived the process; a later scan would find a dead orphan")
-	}
+	eventually(t, 3*time.Second,
+		"the state file outlived the process; a later scan would find a dead orphan", func() bool {
+			_, err := os.Stat(s.statePath("a"))
+			return os.IsNotExist(err)
+		})
 }
 
 // --- state machine and ring buffer ---
@@ -743,7 +748,9 @@ func TestWaitAliveFailsOnEarlyExit(t *testing.T) {
 	if err := s.WaitAlive(t.Context(), sh); err == nil {
 		t.Fatal("a worker that died immediately was reported ready")
 	}
-	if d := time.Since(start); d > 1500*time.Millisecond {
+	// Below AliveGrace (5s), which is what waiting it out would take; the
+	// headroom is for a loaded runner, where 1.5s was sometimes not enough.
+	if d := time.Since(start); d > 4*time.Second {
 		t.Errorf("waited %v for a process that had already exited", d)
 	}
 }
@@ -794,3 +801,50 @@ func TestAnAdoptedShardCanBecomeReady(t *testing.T) {
 	}
 	_ = s2.Stop(t.Context(), "a")
 }
+
+func TestAWorkersOnlyProcessWaitsForTheLeader(t *testing.T) {
+	// A Ray worker's join is its only process. The wait was honoured only for
+	// sidecars, so the join started at once, found no head, exited, and took
+	// the replica down with it.
+	s, _ := newSupervisor(t)
+	s.cfg.RendezvousTimeout = 5 * time.Second
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	port := ln.Addr().(*net.TCPAddr).Port
+	_ = ln.Close()
+	go func() {
+		time.Sleep(400 * time.Millisecond)
+		l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+		if err == nil {
+			t.Cleanup(func() { _ = l.Close() })
+		}
+	}()
+
+	spec := engines.ShardSpec{Rank: 1, LeaderAddr: "127.0.0.1", LeaderPort: port}
+	cmds := []engines.Command{{Name: "join", Path: "/bin/sh", Args: []string{"-c", "sleep 30"},
+		Env: os.Environ(), WaitForLeaderPort: true}}
+	start := time.Now()
+	if _, err := s.Start(t.Context(), "w", spec, cmds); err != nil {
+		t.Fatal(err)
+	}
+	if d := time.Since(start); d < 300*time.Millisecond {
+		t.Errorf("the worker started after %v, before the leader was listening", d)
+	}
+}
+
+// eventually waits for cond, failing with msg if it does not hold within
+// limit. Sleeping a fixed time and then checking measured the machine the
+// test ran on, and a loaded CI runner under the race detector lost that race.
+func eventually(t *testing.T, limit time.Duration, msg string, cond func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(limit)
+	for !cond() {
+		if time.Now().After(deadline) {
+			t.Fatal(msg)
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+}

internal/agent/upgrade.go

@@ -0,0 +1,58 @@
+package agent
+
+import (
+	"context"
+	"os"
+	"runtime"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/release"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+)
+
+// upgradeExit is the exit status after an upgrade: non-zero, so systemd's
+// Restart=on-failure starts the new binary, and distinct, so the journal says
+// why the agent went.
+const upgradeExit = 75
+
+// exitForUpgrade ends the process; a variable so a test can see it called
+// rather than have its own process end.
+var exitForUpgrade = func() { os.Exit(upgradeExit) }
+
+// upgradeTarget is the binary an upgrade replaces; a variable so a test
+// replaces a file of its own rather than the test binary running it.
+var upgradeTarget = os.Executable
+
+// upgrade replaces this agent's binary with a release and restarts into it.
+//
+// Engines are not touched: an agent restart leaves them running and adopts
+// them again, which is what makes upgrading a node something that can be done
+// while it serves.
+func (a *Agent) upgrade(ctx context.Context, m *llmgwv1.UpgradeAgent, out chan<- *llmgwv1.AgentMessage) {
+	reply := &llmgwv1.UpgradeReply{RequestId: m.GetRequestId(), Version: m.GetVersion()}
+	send := func() {
+		select {
+		case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: reply}}:
+		case <-ctx.Done():
+		}
+	}
+	a.log.Info("upgrading", "to", m.GetVersion(), "from", a.cfg.Version)
+	self, err := upgradeTarget()
+	if err == nil {
+		dctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
+		_, err = release.Install(dctx, nil, m.GetReleases(), m.GetVersion(), runtime.GOOS, runtime.GOARCH, self)
+		cancel()
+	}
+	if err != nil {
+		reply.Error = err.Error()
+		a.log.Error("upgrade failed; still running the old version", "to", m.GetVersion(), "err", err)
+		send()
+		return
+	}
+	send()
+	a.log.Info("upgraded; restarting into the new version", "to", m.GetVersion())
+	// A moment for the reply to leave before the stream goes.
+	time.Sleep(500 * time.Millisecond)
+	exitForUpgrade()
+}

internal/agent/upgrade_test.go

@@ -0,0 +1,78 @@
+package agent
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/release"
+)
+
+func TestAnAgentUpgradesItselfAndSaysSoBeforeRestarting(t *testing.T) {
+	body := []byte("the new agent")
+	sum := sha256.Sum256(body)
+	asset := release.AssetName("v9.9.9", runtime.GOOS, runtime.GOARCH)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/v9.9.9/checksums.txt":
+			_, _ = w.Write([]byte(hex.EncodeToString(sum[:]) + "  " + asset + "\n"))
+		case "/v9.9.9/" + asset:
+			_, _ = w.Write(body)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+
+	bin := filepath.Join(t.TempDir(), "llmgw")
+	_ = os.WriteFile(bin, []byte("the old agent"), 0o755)
+	exited := make(chan struct{}, 1)
+	oldTarget, oldExit := upgradeTarget, exitForUpgrade
+	upgradeTarget = func() (string, error) { return bin, nil }
+	exitForUpgrade = func() { exited <- struct{}{} }
+	t.Cleanup(func() { upgradeTarget, exitForUpgrade = oldTarget, oldExit })
+
+	cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(), EnginesDir: t.TempDir()}
+	cfg.Superv.WorkDir = t.TempDir()
+	a, err := New(cfg, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	out := make(chan *llmgwv1.AgentMessage, 1)
+	go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u1", Version: "v9.9.9", Releases: srv.URL}, out)
+
+	select {
+	case m := <-out:
+		if r := m.GetUpgrade(); r.GetRequestId() != "u1" || r.GetError() != "" {
+			t.Fatalf("reply %+v", r)
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("no reply")
+	}
+	select {
+	case <-exited:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the agent did not restart into the new binary")
+	}
+	if b, _ := os.ReadFile(bin); string(b) != "the new agent" {
+		t.Errorf("binary holds %q", b)
+	}
+
+	// A version with no release answers with why, and does not restart.
+	go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u2", Version: "v9.9.8", Releases: srv.URL}, out)
+	if r := (<-out).GetUpgrade(); r.GetError() == "" {
+		t.Error("an upgrade to a release that does not exist reported success")
+	}
+	select {
+	case <-exited:
+		t.Error("a failed upgrade restarted the agent")
+	case <-time.After(time.Second):
+	}
+}

internal/agent/weights/set.go

@@ -2,6 +2,8 @@ package weights
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -26,8 +28,9 @@ type manifest struct {
 	Model   string `json:"model"`
 	Version string `json:"version"`
 	Files   []struct {
-		Name  string `json:"name"`
-		Bytes int64  `json:"bytes"`
+		Name   string `json:"name"`
+		Bytes  int64  `json:"bytes"`
+		SHA256 string `json:"sha256"`
 	} `json:"files"`
 	Bytes int64 `json:"bytes"`
 }
@@ -87,6 +90,25 @@ func (f *Fetcher) ensureSet(ctx context.Context, source string) (string, error)
 		}
 	}
 
+	// Each file against the checksum the mirror took when it fetched it,
+	// before the set is called complete: a shard damaged on the way loads as
+	// a model that answers nonsense, not as an error.
+	for _, file := range m.Files {
+		if file.SHA256 == "" {
+			continue
+		}
+		target := filepath.Join(dir, file.Name)
+		got, err := fileSHA256(target)
+		if err != nil {
+			return "", err
+		}
+		if !strings.EqualFold(got, file.SHA256) {
+			_ = os.Remove(target)
+			return "", fmt.Errorf("%s%s: sha256 %s, the mirror has %s; removed, to be fetched again",
+				source, file.Name, got, file.SHA256)
+		}
+	}
+
 	if err := os.WriteFile(done, []byte(source+"\n"), 0o644); err != nil {
 		return "", err
 	}
@@ -108,6 +130,12 @@ func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, err
 		return manifest{}, fmt.Errorf("asking %s what it holds: %w", source, err)
 	}
 	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusConflict {
+		// Still downloading on the mirror, or a shard missing: said in the
+		// mirror's words, which name what is not there yet.
+		why, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
+		return manifest{}, fmt.Errorf("%s is not complete on the mirror yet: %s", source, strings.TrimSpace(string(why)))
+	}
 	if resp.StatusCode != http.StatusOK {
 		return manifest{}, fmt.Errorf("%s answered %d", source, resp.StatusCode)
 	}
@@ -127,3 +155,16 @@ func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, err
 	}
 	return m, nil
 }
+
+func fileSHA256(path string) (string, error) {
+	f, err := os.Open(path) //nolint:gosec // a file this fetcher wrote
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(h.Sum(nil)), nil
+}

internal/agent/weights/set_test.go

@@ -1,6 +1,8 @@
 package weights
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -165,3 +167,40 @@ func TestOnlyATrailingSlashMeansASet(t *testing.T) {
 		}
 	}
 }
+
+func TestASetFileThatDoesNotMatchTheMirrorsChecksumIsRefused(t *testing.T) {
+	good := "the shard as mirrored"
+	sum := sha256.Sum256([]byte(good))
+	served := "the shard damaged on the way"
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasSuffix(r.URL.Path, "/") {
+			fmt.Fprintf(w, `{"model":"m","version":"v","files":[{"name":"s.safetensors","bytes":%d,"sha256":%q}]}`,
+				len(served), hex.EncodeToString(sum[:]))
+			return
+		}
+		_, _ = w.Write([]byte(served))
+	}))
+	t.Cleanup(srv.Close)
+	f := &Fetcher{CacheDir: t.TempDir()}
+	if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err == nil || !strings.Contains(err.Error(), "sha256") {
+		t.Fatalf("a damaged shard was accepted: %v", err)
+	}
+	served = good
+	// The size differs, so it is fetched again; this time it matches.
+	if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil {
+		t.Fatalf("the shard as mirrored: %v", err)
+	}
+}
+
+func TestASetStillDownloadingOnTheMirrorSaysSo(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		http.Error(w, "m/v: this version is not complete: still downloading model-00002-of-00002.safetensors", http.StatusConflict)
+	}))
+	t.Cleanup(srv.Close)
+	f := &Fetcher{CacheDir: t.TempDir()}
+	_, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
+	if err == nil || !strings.Contains(err.Error(), "not complete on the mirror yet") ||
+		!strings.Contains(err.Error(), "model-00002-of-00002") {
+		t.Fatalf("got %v", err)
+	}
+}

internal/auth/auth.go

@@ -21,6 +21,9 @@ import (
 type Principal struct {
 	KeyID     string
 	KeyPrefix string
+	// UserID is the person the key belongs to, empty for a key an operator
+	// issued in a project. A personal key has a person and no project.
+	UserID    string
 	ProjectID string
 	TeamID    string
 	OrgID     string
@@ -186,7 +189,7 @@ func (a *Authenticator) resolve(ctx context.Context, hash []byte) (Principal, er
 		return Principal{}, err
 	}
 	return Principal{
-		KeyID: k.ID, KeyPrefix: k.Prefix, ProjectID: eff.ProjectID,
+		KeyID: k.ID, KeyPrefix: k.Prefix, UserID: eff.UserID, ProjectID: eff.ProjectID,
 		TeamID: eff.TeamID, OrgID: eff.OrgID,
 		Band: eff.Band, RoutingProfile: eff.RoutingProfile,
 		Limits: eff,

internal/bench/bench.go

@@ -225,6 +225,7 @@ func runStep(ctx context.Context, client *http.Client, cfg Config, concurrency i
 	)
 
 	sampler := newSampler(cfg.MetricsURL, cfg.Engine)
+	sampler.apiKey = cfg.APIKey
 	sampler.start(ctx)
 
 	start := time.Now()

internal/bench/sampler.go

@@ -19,6 +19,7 @@ import (
 // Optional, because a run pointed at the gateway has no single engine to read.
 type sampler struct {
 	url    string
+	apiKey string
 	engine string
 	client *http.Client
 	cancel context.CancelFunc
@@ -65,6 +66,9 @@ func (s *sampler) sample(ctx context.Context) {
 	if err != nil {
 		return
 	}
+	if s.apiKey != "" {
+		req.Header.Set("Authorization", "Bearer "+s.apiKey)
+	}
 	resp, err := s.client.Do(req)
 	if err != nil {
 		return

internal/cli/agent.go

@@ -163,10 +163,10 @@ func printInventory(inv discovery.Inventory) {
 				g.Index, g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health)
 		}
 		fmt.Fprintf(w, "  classes: %v\n", inv.Classes())
-		if !discovery.NVMLSupported() {
-			fmt.Fprintln(w, "  NVLink topology unavailable: rebuild with -tags nvml.")
-			fmt.Fprintln(w, "  Without it, placement cannot keep a tensor-parallel group inside")
-			fmt.Fprintln(w, "  one NVLink domain (plan §6.2).")
+		for _, g := range inv.GPUs {
+			if len(g.NVLinkPeers) > 0 {
+				fmt.Fprintf(w, "  GPU %d NVLink peers: %v\n", g.Index, g.NVLinkPeers)
+			}
 		}
 	}
 

internal/cli/backup.go

@@ -0,0 +1,330 @@
+package cli
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/version"
+)
+
+// DefaultBackupDir is where backups are written and where the control plane
+// looks for the newest one.
+//
+// Its own directory rather than /var/backups/llmgw, which a PostgreSQL move
+// had already made for its own dumps, owned by postgres and closed to
+// everyone else — so the control plane could not see a backup landing in it.
+const DefaultBackupDir = "/var/backups/llmgw-server"
+
+func newBackupCmd() *cobra.Command {
+	var (
+		configPath string
+		etcDir     string
+		dir        string
+		keep       int
+	)
+	c := &cobra.Command{
+		Use:   "backup",
+		Short: "Write the store and /etc/llmgw to one archive, and keep the last few",
+		Long: "Everything a control plane cannot be rebuilt without: the store — keys,\n" +
+			"people, quotas, usage, the models added at runtime, the secret engine\n" +
+			"keys derive from — and /etc/llmgw, which holds the configuration, the\n" +
+			"store's address and the certificate authority's key.\n\n" +
+			"The archive holds that key, so it is written readable by root only. It\n" +
+			"is on the same disk as what it backs up: copy it elsewhere, or it does\n" +
+			"not survive the loss of this machine.\n\n" +
+			"Run daily by llmgw-backup.timer (`llmgw server backup`). The control plane publishes the time of\n" +
+			"the newest archive as llmgw_backup_last_success_timestamp_seconds, and\n" +
+			"BackupTooOld fires when it is more than a day old.",
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			cfg, err := config.Load(configPath)
+			if err != nil {
+				return err
+			}
+			path, err := writeBackup(cmd.Context(), cfg.Auth.Store, etcDir, dir)
+			if err != nil {
+				return err
+			}
+			if why := unlistable(dir); why != "" {
+				fmt.Fprintf(cmd.ErrOrStderr(), "warning: %s; the control plane will report no backup "+
+					"and BackupTooOld will fire\n", why)
+			}
+			removed, err := pruneBackups(dir, keep)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintf(cmd.OutOrStdout(), "wrote %s", path)
+			if removed > 0 {
+				fmt.Fprintf(cmd.OutOrStdout(), "; removed %d older", removed)
+			}
+			fmt.Fprintln(cmd.OutOrStdout())
+			return nil
+		},
+	}
+	c.Flags().StringVar(&configPath, "config", "/etc/llmgw/server.yaml", "the control plane's configuration")
+	c.Flags().StringVar(&etcDir, "etc", "/etc/llmgw", "configuration directory to include")
+	c.Flags().StringVar(&dir, "dir", DefaultBackupDir, "where archives are written")
+	c.Flags().IntVar(&keep, "keep", 14, "how many archives to keep")
+	return c
+}
+
+const backupPrefix = "llmgw-backup-"
+
+// writeBackup writes one archive and returns its path. Written beside its
+// final name and renamed, so a backup interrupted halfway is never the newest
+// one the control plane reports.
+func writeBackup(ctx context.Context, storeAddr, etcDir, dir string) (string, error) {
+	// Listable by anyone, so the control plane — not root — can see when the
+	// newest archive was written. The archives themselves are root's alone.
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return "", err
+	}
+	host, _ := os.Hostname()
+	name := backupPrefix + host + "-" + time.Now().UTC().Format("20060102T150405Z") + ".tar.gz"
+	final := filepath.Join(dir, name)
+	part := final + ".part"
+
+	f, err := os.OpenFile(part, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
+	if err != nil {
+		return "", err
+	}
+	ok := false
+	defer func() {
+		if !ok {
+			_ = os.Remove(part)
+		}
+	}()
+	gz := gzip.NewWriter(f)
+	tw := tar.NewWriter(gz)
+
+	manifest := fmt.Sprintf("llmgw %s\nhost %s\ntime %s\n", version.Version, host,
+		time.Now().UTC().Format(time.RFC3339))
+	if err := addBytes(tw, "MANIFEST", []byte(manifest)); err != nil {
+		return "", err
+	}
+	if storeAddr != "" {
+		if err := addStore(ctx, tw, storeAddr, dir); err != nil {
+			return "", fmt.Errorf("the store: %w", err)
+		}
+	}
+	if err := addTree(tw, etcDir, "etc/llmgw"); err != nil {
+		return "", fmt.Errorf("%s: %w", etcDir, err)
+	}
+	if err := errors.Join(tw.Close(), gz.Close(), f.Sync(), f.Close()); err != nil {
+		return "", err
+	}
+	if err := os.Rename(part, final); err != nil {
+		return "", err
+	}
+	ok = true
+	return final, nil
+}
+
+// addStore dumps the store into the archive.
+func addStore(ctx context.Context, tw *tar.Writer, addr, scratch string) error {
+	addr, err := store.ResolveAddress(addr)
+	if err != nil {
+		return err
+	}
+	// Inside a directory only its owner can enter. The archive directory is
+	// listable on purpose, and VACUUM INTO creates its file with the default
+	// mode: made there directly, a full copy of the store — password hashes,
+	// the engine-key secret — sat readable by any local user for as long as
+	// the backup took.
+	tmpDir, err := os.MkdirTemp(scratch, ".store-")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpDir)
+	tmpPath := filepath.Join(tmpDir, "store")
+
+	name := "store.sqlite"
+	if store.IsPostgres(addr) {
+		name = "store.pgdump"
+		if err := pgDump(ctx, addr, tmpPath); err != nil {
+			return err
+		}
+	} else {
+		db, err := store.Open(ctx, addr)
+		if err != nil {
+			return err
+		}
+		err = db.SnapshotTo(ctx, tmpPath)
+		_ = db.Close()
+		if err != nil {
+			return err
+		}
+	}
+	return addFile(tw, tmpPath, name)
+}
+
+// pgDump runs pg_dump in its custom format, which pg_restore can restore
+// selectively. The connection is given through the environment, not the
+// command line: a DSN carries the password, and every user on the machine
+// can read another process's arguments.
+func pgDump(ctx context.Context, dsn, out string) error {
+	vars, err := pgEnv(dsn)
+	if err != nil {
+		return err
+	}
+	cmd := exec.CommandContext(ctx, "pg_dump", "--format=custom", "--no-owner", "--file="+out) //nolint:gosec // fixed binary, our own file
+	cmd.Env = append(os.Environ(), vars...)
+	if b, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("pg_dump: %v: %s", err, strings.TrimSpace(string(b)))
+	}
+	return nil
+}
+
+// pgEnv turns a store address into libpq's environment. Several hosts — the
+// members of a replicated cluster — become libpq's own lists in PGHOST and
+// PGPORT; read as one URL host, the list was one name libpq could not
+// resolve, and a backup worked only by falling through to the second.
+func pgEnv(dsn string) ([]string, error) {
+	hosts, ports, u, err := store.SplitPostgres(dsn)
+	if err != nil {
+		return nil, err
+	}
+	var env []string
+	set := func(k, v string) {
+		if v != "" {
+			env = append(env, k+"="+v)
+		}
+	}
+	set("PGHOST", strings.Join(hosts, ","))
+	set("PGPORT", strings.Join(ports, ","))
+	set("PGUSER", u.User.Username())
+	if pw, ok := u.User.Password(); ok {
+		set("PGPASSWORD", pw)
+	}
+	set("PGDATABASE", strings.TrimPrefix(u.Path, "/"))
+	set("PGSSLMODE", u.Query().Get("sslmode"))
+	// A standby if there is one, else whichever answers — not the address's
+	// read-write: a dump only reads, it need not load the primary, and one
+	// that insists on the primary fails exactly while the cluster is failing
+	// over.
+	if len(hosts) > 1 {
+		set("PGTARGETSESSIONATTRS", "prefer-standby")
+	}
+	return env, nil
+}
+
+func addTree(tw *tar.Writer, root, as string) error {
+	return filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		rel, _ := filepath.Rel(root, p)
+		name := filepath.ToSlash(filepath.Join(as, rel))
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+		switch {
+		case d.IsDir():
+			return tw.WriteHeader(&tar.Header{Name: name + "/", Mode: int64(info.Mode().Perm()),
+				Typeflag: tar.TypeDir, ModTime: info.ModTime()})
+		case info.Mode().IsRegular():
+			return addFile(tw, p, name)
+		}
+		return nil // sockets, links: not configuration
+	})
+}
+
+func addFile(tw *tar.Writer, path, name string) error {
+	f, err := os.Open(path) //nolint:gosec // files we chose to back up
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	info, err := f.Stat()
+	if err != nil {
+		return err
+	}
+	if err := tw.WriteHeader(&tar.Header{Name: name, Mode: int64(info.Mode().Perm()),
+		Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil {
+		return err
+	}
+	_, err = io.Copy(tw, f)
+	return err
+}
+
+func addBytes(tw *tar.Writer, name string, b []byte) error {
+	if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b)),
+		ModTime: time.Now(), Typeflag: tar.TypeReg}); err != nil {
+		return err
+	}
+	_, err := tw.Write(b)
+	return err
+}
+
+// pruneBackups keeps the newest keep archives. Only files this command
+// wrote are considered, so nothing else in the directory is ever removed.
+func pruneBackups(dir string, keep int) (int, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return 0, err
+	}
+	var names []string
+	for _, e := range entries {
+		if n := e.Name(); strings.HasPrefix(n, backupPrefix) && strings.HasSuffix(n, ".tar.gz") {
+			names = append(names, n)
+		}
+	}
+	// The name carries a UTC timestamp, so name order is age order.
+	slices.Sort(names)
+	removed := 0
+	for len(names) > max(keep, 1) {
+		if err := os.Remove(filepath.Join(dir, names[0])); err != nil {
+			return removed, err
+		}
+		names = names[1:]
+		removed++
+	}
+	return removed, nil
+}
+
+// NewestBackup is the modification time of the newest complete archive in
+// dir, or zero when there is none.
+func NewestBackup(dir string) time.Time {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return time.Time{}
+	}
+	var newest time.Time
+	for _, e := range entries {
+		if n := e.Name(); !strings.HasPrefix(n, backupPrefix) || !strings.HasSuffix(n, ".tar.gz") {
+			continue
+		}
+		if info, err := e.Info(); err == nil && info.ModTime().After(newest) {
+			newest = info.ModTime()
+		}
+	}
+	return newest
+}
+
+// unlistable says why the control plane, which does not run as root, could
+// not see the archives in dir, or "" when it can.
+func unlistable(dir string) string {
+	info, err := os.Stat(dir)
+	if err != nil {
+		return err.Error()
+	}
+	if info.Mode().Perm()&0o005 != 0o005 {
+		return fmt.Sprintf("%s is %v, so only its owner can list it", dir, info.Mode().Perm())
+	}
+	return ""
+}

internal/cli/backup_test.go

@@ -0,0 +1,164 @@
+package cli
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+func TestABackupHoldsTheStoreAndTheConfiguration(t *testing.T) {
+	tmp := t.TempDir()
+	dbPath := filepath.Join(tmp, "store.db")
+	db, err := store.Open(t.Context(), dbPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	secret, err := db.ClusterSecret(t.Context(), "engine-key")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = db.Close()
+
+	etc := filepath.Join(tmp, "etc")
+	_ = os.MkdirAll(filepath.Join(etc, "pki"), 0o700)
+	_ = os.WriteFile(filepath.Join(etc, "server.yaml"), []byte("listen: :8080\n"), 0o644)
+	_ = os.WriteFile(filepath.Join(etc, "pki", "ca-key.pem"), []byte("KEY"), 0o600)
+
+	out := filepath.Join(tmp, "backups")
+	path, err := writeBackup(t.Context(), dbPath, etc, out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	info, _ := os.Stat(path)
+	if info.Mode().Perm() != 0o600 {
+		t.Errorf("the archive holds the CA's key and is %v", info.Mode().Perm())
+	}
+
+	files := map[string][]byte{}
+	f, _ := os.Open(path)
+	defer f.Close()
+	gz, err := gzip.NewReader(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tr := tar.NewReader(gz)
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		b, _ := io.ReadAll(tr)
+		files[h.Name] = b
+	}
+	for _, want := range []string{"MANIFEST", "store.sqlite", "etc/llmgw/server.yaml", "etc/llmgw/pki/ca-key.pem"} {
+		if _, ok := files[want]; !ok {
+			t.Errorf("%s is not in the archive", want)
+		}
+	}
+
+	// And the store in it is a working store, with what was in it.
+	restored := filepath.Join(tmp, "restored.db")
+	_ = os.WriteFile(restored, files["store.sqlite"], 0o600)
+	rdb, err := store.Open(t.Context(), restored)
+	if err != nil {
+		t.Fatalf("the backed-up store does not open: %v", err)
+	}
+	defer rdb.Close()
+	if got, _ := rdb.ClusterSecret(t.Context(), "engine-key"); got != secret {
+		t.Error("the restored store lost the engine key secret; every engine would be locked out")
+	}
+}
+
+func TestOldBackupsArePrunedAndTheNewestIsFound(t *testing.T) {
+	dir := t.TempDir()
+	for i, stamp := range []string{"20260101T000000Z", "20260102T000000Z", "20260103T000000Z"} {
+		p := filepath.Join(dir, backupPrefix+"h-"+stamp+".tar.gz")
+		_ = os.WriteFile(p, []byte("x"), 0o600)
+		mt := time.Date(2026, 1, 1+i, 0, 0, 0, 0, time.UTC)
+		_ = os.Chtimes(p, mt, mt)
+	}
+	_ = os.WriteFile(filepath.Join(dir, "someone-elses-file"), []byte("x"), 0o600)
+	_ = os.WriteFile(filepath.Join(dir, backupPrefix+"h-20260104T000000Z.tar.gz.part"), []byte("x"), 0o600)
+
+	removed, err := pruneBackups(dir, 2)
+	if err != nil || removed != 1 {
+		t.Fatalf("removed %d, %v", removed, err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "someone-elses-file")); err != nil {
+		t.Error("a file the backup did not write was removed")
+	}
+	newest := NewestBackup(dir)
+	if !newest.Equal(time.Date(2026, 1, 3, 0, 0, 0, 0, time.UTC)) {
+		t.Errorf("newest %v; an interrupted .part must not count", newest)
+	}
+	if !NewestBackup(filepath.Join(dir, "none")).IsZero() {
+		t.Error("a missing directory reported a backup")
+	}
+}
+
+func TestABackupDirectoryTheControlPlaneCannotListIsSaid(t *testing.T) {
+	dir := t.TempDir()
+	_ = os.Chmod(dir, 0o700)
+	if unlistable(dir) == "" {
+		t.Error("a directory only its owner can list was taken as visible")
+	}
+	_ = os.Chmod(dir, 0o755)
+	if why := unlistable(dir); why != "" {
+		t.Errorf("a listable directory: %s", why)
+	}
+}
+
+func TestNoScratchCopyOfTheStoreIsLeftBehind(t *testing.T) {
+	// The snapshot is made before it is archived. Made straight into the
+	// listable archive directory, it was readable by anyone for that time.
+	tmp := t.TempDir()
+	dbPath := filepath.Join(tmp, "store.db")
+	db, err := store.Open(t.Context(), dbPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = db.Close()
+	out := filepath.Join(tmp, "backups")
+	if _, err := writeBackup(t.Context(), dbPath, t.TempDir(), out); err != nil {
+		t.Fatal(err)
+	}
+	entries, _ := os.ReadDir(out)
+	for _, e := range entries {
+		if strings.HasPrefix(e.Name(), ".store") {
+			t.Errorf("a scratch copy of the store was left behind: %s", e.Name())
+		}
+	}
+}
+
+func TestAStoreOfSeveralHostsIsGivenToPgDumpAsLibpqLists(t *testing.T) {
+	// The address of a Patroni cluster: url.Parse read its host as one name,
+	// and pg_dump reached the database only by falling through to the second.
+	env, err := pgEnv("postgres://llmgw:s3cret@10.0.0.149:5432,10.0.0.150/llmgw?sslmode=disable&target_session_attrs=read-write")
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := strings.Join(env, " ")
+	for _, want := range []string{"PGHOST=10.0.0.149,10.0.0.150", "PGPORT=5432,5432", "PGUSER=llmgw",
+		"PGPASSWORD=s3cret", "PGDATABASE=llmgw", "PGSSLMODE=disable", "PGTARGETSESSIONATTRS=prefer-standby"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("%s is not in %s", want, got)
+		}
+	}
+	one, err := pgEnv("postgres://u:p@db.internal:6432/llmgw")
+	if err != nil || !strings.Contains(strings.Join(one, " "), "PGHOST=db.internal PGPORT=6432") {
+		t.Errorf("one host: %v, %v", one, err)
+	}
+	if _, err := pgEnv("postgres://u:p@a:5432,:5432/x"); err == nil {
+		t.Error("an empty host in the list was accepted")
+	}
+}

internal/cli/bench.go

@@ -41,6 +41,11 @@ func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 			cfg.MetricsURL = metrics
 			cfg.Engine = engine
 			cfg.APIKey = key
+			// A replica's engine demands its own key. Asked for rather than
+			// typed: the admin API knows which replica answers at the target.
+			if cfg.APIKey == "" {
+				cfg.APIKey = replicaKey(cmd, client, target)
+			}
 			if len(steps) > 0 {
 				cfg.Concurrency = steps
 			}
@@ -227,3 +232,18 @@ func ms(d time.Duration) string {
 		return d.Round(10 * time.Millisecond).String()
 	}
 }
+
+// replicaKey is the engine key of the replica at target, or "" when target is
+// not a replica (the gateway, say) or the admin API cannot be asked.
+func replicaKey(cmd *cobra.Command, client func() *ctlClient, target string) string {
+	var reps []adminapi.ReplicaView
+	if err := client().get(cmd.Context(), "/api/v1/replicas", &reps); err != nil {
+		return ""
+	}
+	for _, rep := range reps {
+		if sameEndpoint(rep.Endpoint, target) {
+			return rep.EngineKey
+		}
+	}
+	return ""
+}

internal/cli/ctl.go

@@ -67,7 +67,9 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any)
 	if err != nil {
 		return err
 	}
-	if body != nil {
+	// On every change, body or not: the admin listener refuses anything else,
+	// because a browser can send a bodiless POST across origins unasked.
+	if method != http.MethodGet {
 		req.Header.Set("Content-Type", "application/json")
 	}
 	// Who made the change, kept with a stored model so a catalog that grew
@@ -81,7 +83,10 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any)
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode != http.StatusOK {
+	// Any 2xx. The API answers a change it has queued rather than made — a
+	// restart — with 202, and treating that as a failure printed
+	// "Error: 202 Accepted" over a request that had worked.
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
 		var e struct {
 			Error string `json:"error"`
 		}
@@ -117,9 +122,11 @@ func newCtlCmd() *cobra.Command {
 	cmd.AddCommand(
 		newNodesCmd(client, &asJSON),
 		newLogsCmd(client, &asJSON),
+		newRequestCmd(client, &asJSON),
 		newModelsCmd(client, &asJSON),
 		newReplicasCmd(client, &asJSON),
 		newExplainCmd(client, &asJSON),
+		newFabricTestCmd(client, &asJSON),
 		newNodeStateCmd(client, "cordon", "cordoned",
 			"Stop placing new work on a node; what runs there keeps running"),
 		newNodeStateCmd(client, "uncordon", "ready",
@@ -130,12 +137,17 @@ func newCtlCmd() *cobra.Command {
 		// apply to it; it lives here because this is where operators look.
 		newBenchCmd(client, &asJSON),
 		newStatusCmd(client, &asJSON),
-		newModelCmd(client),
+		newModelCmd(client, &asJSON),
 		newScaleCmd(client),
+		newRestartCmd(client),
+		newRetryCmd(client),
+		newUpgradeCmd(client),
 		newEngineCheckCmd(),
+		newEngineBuildCmd(client),
 		newToolCheckCmd(),
 		newWeightsCmd(client, &asJSON),
 		newCapacityCmd(client, &asJSON),
+		newQuotaCmd(client, &asJSON),
 		newUsageCmd(client, &asJSON),
 		newCertCmd(),
 	)
@@ -199,7 +211,8 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 					said = true
 				}
 				for _, p := range n.Provision {
-					if p.State == "ready" {
+					// Skipped is vLLM on a node with no GPU: as it should be.
+					if p.State == "ready" || p.State == "skipped" {
 						continue
 					}
 					fmt.Printf("\n%s: %s %s %s: %s", n.ID, p.Engine, p.Version, p.State, p.Detail)
@@ -240,6 +253,14 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 // actually served, never what is merely catalogued.
 func modelStatus(m adminapi.ModelView) string {
 	switch {
+	case m.Parked:
+		return "parked"
+	case m.External != "":
+		return "external: " + m.External
+	case m.HeldFor != "":
+		return "gave room to " + m.HeldFor
+	case m.Unplaceable:
+		return "cannot place"
 	case m.Min == 0 && m.Total == 0:
 		return "stopped"
 	case m.Ready >= m.Min && m.Min > 0:
@@ -308,17 +329,27 @@ func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 			}
 
 			w := tab()
-			fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tALIASES\tPROFILES\tFALLBACKS")
+			fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tCLASS\tRESERVES\tALIASES\tPROFILES\tFALLBACKS")
+			var warnings []string
 			for _, m := range models {
 				serving := "no"
 				if m.Ready > 0 {
 					serving = "yes"
 				}
-				fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\n",
-					m.Name, serving, m.Ready, m.Min, modelStatus(m),
+				fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\t%s\t%s\n",
+					m.Name, serving, m.Ready, m.Min, modelStatus(m), m.Class, reserves(m),
 					join(m.Aliases), join(m.Profiles), join(m.Fallbacks))
+				for _, x := range m.Warnings {
+					warnings = append(warnings, m.Name+": "+x)
+				}
 			}
-			return w.Flush()
+			if err := w.Flush(); err != nil {
+				return err
+			}
+			for _, x := range warnings {
+				fmt.Println("warning:", x)
+			}
+			return nil
 		},
 	}
 }
@@ -366,6 +397,9 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 				if r.Stale {
 					state += " (stale profile)"
 				}
+				if r.Restart != "" {
+					state += " (restart " + r.Restart + ")"
+				}
 				fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
 					r.ID, r.Model, state, r.Pool, join(nodes),
 					(time.Duration(r.AgeSec) * time.Second).String(),
@@ -470,8 +504,12 @@ func newNodeStateCmd(client func() *ctlClient, use, state, short string) *cobra.
 
 // --- output helpers ---
 
-func tab() *tabwriter.Writer {
-	return tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+func tab() *tabwriter.Writer { return tabTo(os.Stdout) }
+
+// tabTo is tab writing where it is told: a summary drawn into the console
+// went to the terminal underneath it instead, every three seconds.
+func tabTo(w io.Writer) *tabwriter.Writer {
+	return tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
 }
 
 func emitJSON(v any) error {
@@ -493,3 +531,22 @@ func dash(s string) string {
 	}
 	return s
 }
+
+// reserves is what one replica of a model holds of a node, with what it was
+// measured using beside it: "6G 2c (5.1G used)".
+func reserves(m adminapi.ModelView) string {
+	if m.MemoryGB == 0 && m.CPUCores == 0 {
+		return "-"
+	}
+	out := fmt.Sprintf("%dG", m.MemoryGB)
+	if m.CPUCores > 0 {
+		out += fmt.Sprintf(" %dc", m.CPUCores)
+	}
+	if m.MeasuredMemoryGB > 0 {
+		out += fmt.Sprintf(" (%.1fG used)", m.MeasuredMemoryGB)
+	}
+	if len(m.Warnings) > 0 {
+		out += " !"
+	}
+	return out
+}

internal/cli/ctl_fabrictest.go

@@ -0,0 +1,75 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"text/tabwriter"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+)
+
+func newFabricTestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
+	var (
+		fabric  string
+		seconds int
+	)
+	c := &cobra.Command{
+		Use:   "fabric-test NODE-A NODE-B",
+		Short: "Measure RDMA bandwidth between two nodes, rail by rail",
+		Long: `Runs ib_write_bw from each of NODE-A's adapters on a fabric to the same
+adapter on NODE-B — mlx5_0 to mlx5_0, and so on, the way NCCL pairs them —
+one rail at a time, and grades each against its link's rate: ok at 80% or
+more, slow at 50%, fail below or when nothing got through.
+
+Every check on one node can pass while two nodes still cannot reach each
+other: a cable into the wrong switch, a partition key, a RoCE network with no
+route. A replica spanning them then hangs on its first all-reduce. Run this
+before placing one.
+
+It goes through the agents, so it needs no ssh between nodes. perftest is
+installed on a node the first time it is asked. The traffic is real and
+takes a few seconds a rail; run it when the fabric is quiet.`,
+		Example: `  llmgw ctl fabric-test gpu01 gpu02
+  llmgw ctl fabric-test gpu01 gpu02 --fabric ib-compute --seconds 10`,
+		Args: cobra.ExactArgs(2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			body, _ := json.Marshal(map[string]any{"a": args[0], "b": args[1], "fabric": fabric, "seconds": seconds})
+			cl := client()
+			// Rails one after another, each for its seconds and a settle; the
+			// client's usual timeout is for questions, not measurements.
+			cl.http.Timeout = 30 * time.Minute
+			var v adminapi.FabricTestView
+			if err := cl.post(cmd.Context(), "/api/v1/fabric-test", body, &v); err != nil {
+				return err
+			}
+			if *asJSON {
+				return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
+			}
+			out := cmd.OutOrStdout()
+			fmt.Fprintf(out, "%s → %s on %s, %s\n\n", v.A, v.B, v.Fabric, v.Took)
+			w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+			fmt.Fprintln(w, "RAIL\tLINK\tMEASURED\tVERDICT\tDETAIL")
+			for _, r := range v.Rails {
+				link := "-"
+				if r.RateGbps > 0 {
+					link = fmt.Sprintf("%d Gb/s", r.RateGbps)
+				}
+				fmt.Fprintf(w, "%s → %s\t%s\t%.1f Gb/s\t%s\t%s\n", r.A, r.B, link, r.Gbps, r.Verdict, r.Detail)
+			}
+			_ = w.Flush()
+			for _, n := range v.Notes {
+				fmt.Fprintf(out, "\nnote: %s\n", n)
+			}
+			if !v.OK {
+				return fmt.Errorf("not every rail passed")
+			}
+			return nil
+		},
+	}
+	c.Flags().StringVar(&fabric, "fabric", "", "the fabric to test (default: the one the two nodes share)")
+	c.Flags().IntVar(&seconds, "seconds", 3, "how long each rail runs, up to 30")
+	return c
+}

internal/cli/ctl_request.go

@@ -0,0 +1,45 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+)
+
+func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
+	return &cobra.Command{
+		Use:   "request ID",
+		Short: "What one request went through: replica, queue, first token, tokens",
+		Long: "Every response carries its id in X-Request-Id, and a client may send its\n" +
+			"own. Given one, this says which replica answered, how long admission held\n" +
+			"it, how long the first token took and what it cost — the answer to \"why\n" +
+			"was this slow\". The access log has the same fields under \"request\".",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			var v adminapi.RequestView
+			if err := client().get(cmd.Context(), "/api/v1/requests/"+url.PathEscape(args[0]), &v); err != nil {
+				return err
+			}
+			if *asJSON {
+				return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
+			}
+			model := v.Model
+			if v.ServedModel != "" && v.ServedModel != v.Model {
+				model += " (served by " + v.ServedModel + ")"
+			}
+			out := cmd.OutOrStdout()
+			fmt.Fprintf(out, "request   %s at %s\n", v.RequestID, v.At.UTC().Format("2006-01-02 15:04:05Z"))
+			fmt.Fprintf(out, "model     %s\n", model)
+			fmt.Fprintf(out, "replica   %s\n", or(v.Replica, "- (none answered)"))
+			fmt.Fprintf(out, "status    %d\n", v.Status)
+			fmt.Fprintf(out, "time      %d ms total: %d queued, %d to the first token\n", v.LatencyMs, v.QueueMs, v.TTFTMs)
+			fmt.Fprintf(out, "tokens    %d in, %d out\n", v.PromptTokens, v.CompletionTokens)
+			fmt.Fprintf(out, "who       key %s, project %s%s\n", v.KeyID, v.ProjectID, prefixed(", user ", v.UserID))
+			return nil
+		},
+	}
+}