Describe v0.8.5

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

.gitea/workflows/ci.yml

@@ -38,3 +38,11 @@ jobs:
 
       - name: Build
         run: go build ./...
+
+      # The NVML backend needs cgo and cannot run without a driver, but it must
+      # at least keep compiling: it is the only source of NVLink topology and a
+      # break would otherwise surface on a GPU node, far from here.
+      - name: Build the NVML backend
+        env:
+          CGO_ENABLED: "1"
+        run: go build -tags nvml ./...

.gitea/workflows/release.yml

@@ -0,0 +1,180 @@
+# Attaches binaries to a release when a version tag is pushed.
+#
+# Without this a release is a git tag, and installing llmgw on a node means
+# having a Go toolchain and the source. The onboarding script copies a binary
+# from wherever it is run, which works and quietly makes whoever runs it the
+# build server.
+name: Release
+
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    # The built-in token every job is given carries whatever the instance's
+    # default mode allows, and under the restricted default that is read-only
+    # on releases — so the build would succeed and the publish would be
+    # refused. `contents` covers code and releases together, which is what
+    # creating a release needs: `releases: write` alone was refused, because
+    # creating one can create its tag.
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Tags are what the version string is built from, and a shallow clone
+          # has none of them.
+          fetch-depth: 0
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Build
+        run: |
+          set -euo pipefail
+          version="${GITHUB_REF_NAME:-$(git describe --tags --always)}"
+          mkdir -p dist
+
+          # CGO_ENABLED=0 is the whole point: the binary has to run on a node
+          # whose libc is older than the builder's, and llmgw has no C
+          # dependency — modernc.org/sqlite is pure Go precisely so this stays
+          # true.
+          for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64; do
+            os="${target%/*}"; arch="${target#*/}"
+            name="llmgw-${version}-${os}-${arch}"
+            echo "building $name"
+            CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build \
+              -trimpath \
+              -ldflags "-s -w -X gitea.mynodes.xyz/ilker/llmgw/internal/version.Version=${version}" \
+              -o "dist/${name}" ./cmd/llmgw
+            (cd dist && sha256sum "${name}" >> checksums.txt)
+          done
+
+          # An NVML build for nodes with NVLink, which the portable binary
+          # cannot serve.
+          #
+          # NVML is the only source of NVLink topology, its bindings need cgo,
+          # and cgo is exactly what CGO_ENABLED=0 above exists to avoid. So the
+          # portable binary silently falls back to taking the lowest free GPU
+          # indices — right on an SXM box with NVSwitch, and wrong on a PCIe
+          # box where NVLink bridges pair specific cards: a tensor group
+          # straddling two bridges runs several times slower and nothing says
+          # so. Shipping only the portable build meant nobody could have that
+          # topology without building it themselves.
+          #
+          # Dynamically linked against the host's libc by necessity, so it is
+          # published beside the portable one rather than instead of it.
+          if command -v gcc >/dev/null; then
+            nvml_name="llmgw-${version}-linux-amd64-nvml"
+            CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build \
+              -tags nvml -trimpath \
+              -ldflags "-s -w -X gitea.mynodes.xyz/ilker/llmgw/internal/version.Version=${version}" \
+              -o "dist/${nvml_name}" ./cmd/llmgw
+            (cd dist && sha256sum "${nvml_name}" >> checksums.txt)
+          else
+            echo "no C toolchain; skipping the nvml build" >&2
+          fi
+
+          # Proof the artifact runs at all before it is published. It cannot
+          # test the cross-compiled ones, which is why the native build is
+          # checked rather than assumed.
+          ./dist/llmgw-${version}-linux-amd64 --version
+          cat dist/checksums.txt
+
+      - name: Attach to the release
+        env:
+          TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          set -euo pipefail
+          # Gitea gives every job a GITEA_TOKEN, so this is not the usual
+          # "you forgot to add a secret" — an empty one means the job was denied
+          # a token, which is an instance setting rather than a repository one.
+          if [ -z "${TOKEN:-}" ]; then
+            echo "The job was given no GITEA_TOKEN, so nothing can be uploaded."
+            echo "Check Settings -> Actions -> General; the binaries are in dist/."
+            exit 1
+          fi
+          # workflow_dispatch can be fired from a branch, where GITHUB_REF_NAME
+          # is the branch name. Uploading then would create a release called
+          # "main". The binaries are still built and kept as a check that the
+          # release build works; only the publishing is a tag's business.
+          if [ "${GITHUB_REF_TYPE:-}" != "tag" ]; then
+            echo "not a tag (ref=${GITHUB_REF_NAME}); built, not published"
+            exit 0
+          fi
+          tag="${GITHUB_REF_NAME}"
+          api="${GITHUB_SERVER_URL}/api/v1/repos/${GITHUB_REPOSITORY}"
+
+          # Gitea materialises a release row for every pushed tag, so this
+          # nearly always finds one and the create below is for a tag pushed
+          # some other way.
+          #
+          # The id is read with grep rather than sed because sed's .* is greedy:
+          # "id" appears again inside the nested author object, so the old
+          # pattern took the author's id when it matched at all — and it did not
+          # match "id": 13 with a space, which is what actually happened. The
+          # first "id" in a release object is the release's own.
+          release_id() {
+            curl -sS -H "Authorization: token $TOKEN" "$api/releases/tags/$tag" \
+              | grep -o '"id"[[:space:]]*:[[:space:]]*[0-9]*' \
+              | head -1 | grep -o '[0-9]*$'
+          }
+
+          id=$(release_id || true)
+          if [ -z "$id" ]; then
+            echo "no release for $tag yet; creating one"
+            out=$(curl -sS -w '\n%{http_code}' -X POST \
+                  -H "Authorization: token $TOKEN" \
+                  -H "Content-Type: application/json" \
+                  -d "{\"tag_name\":\"$tag\",\"name\":\"$tag\"}" \
+                  "$api/releases")
+            echo "create returned $(printf '%s' "$out" | tail -1)"
+            # Re-read rather than parsing the create response: a 409 means it
+            # was there all along, which is a success for our purposes.
+            id=$(release_id || true)
+            [ -n "$id" ] || printf '%s\n' "$out" | head -5 >&2
+          fi
+          [ -n "$id" ] || { echo "could not find or create the release" >&2; exit 1; }
+          echo "release id $id"
+
+          for f in dist/*; do
+            echo "uploading $(basename "$f")"
+            curl -sS -f -X POST -H "Authorization: token $TOKEN" \
+              -F "attachment=@$f" \
+              "$api/releases/$id/assets?name=$(basename "$f")" >/dev/null \
+              || { echo "upload of $(basename "$f") was refused" >&2; exit 1; }
+          done
+          echo "attached $(ls dist | wc -l) files to $tag"
+
+  # The release page says what changed, copied from CHANGELOG.md. Gitea makes
+  # an empty release for every tag and the job above only attaches files, so
+  # every release until this existed was a list of binaries and nothing else.
+  #
+  # After the build, because that is what guarantees the release exists. Run by
+  # hand from a branch it syncs every version in the changelog, which is how
+  # the releases made before this got their notes.
+  notes:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Copy the changelog onto the release
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          GITEA_INSTANCE: ${{ github.server_url }}
+          GITEA_REPO: ${{ github.repository }}
+        run: |
+          set -euo pipefail
+          if [ "${GITHUB_REF_TYPE:-}" = "tag" ]; then
+            python3 scripts/gitea/release_notes.py --tag "${GITHUB_REF_NAME}"
+          else
+            python3 scripts/gitea/release_notes.py
+          fi

CHANGELOG.md

@@ -8,14 +8,795 @@ is only released once its **exit criterion** is met, not when its code lands.
 | v0.1.0 | 0 | Skeleton and sim-engine | released |
 | v0.2.0 | 1 | Static router: streaming proxy, filter/score/pick | released |
 | v0.3.0 | 1.5 | Admission control: priority bands, saturation detector | released |
-| v0.4.0 | 2 | Agent: GPU/IB discovery, single-node engines | in progress |
-| v0.5.0 | 3 | Control plane: catalog, placement, reconciler | planned |
-| v0.6.0 | 4 | Multi-node replicas and fabric awareness | planned |
+| v0.4.0 | 2 | Agent: GPU/IB discovery, single-node engines | released as part of v0.5.0 |
+| v0.5.0 | 3 | Control plane: catalog, placement, reconciler | released |
+| v0.6.0 | 4 | Multi-node replicas and fabric awareness | released |
+| v0.7.0 | — | Conversations, agent memory, disk management | released |
+| v0.8.0 | — | PostgreSQL, leader election, agent failover | released |
 | v1.0.0 | 5 | Tenancy, prefix affinity, benchmarking | planned |
 
 Until v1.0.0 the minor version is the breaking one: configuration formats and Go
 APIs change between minors without a deprecation period.
 
+## v0.8.5 — 2026-09-28
+
+### A control plane in one command
+
+```
+sudo llmgw server init --admin you@example.com
+```
+
+Creates the llmgw user and its directories, the certificate authority, the
+control plane's certificate for the machine's hostname and every address it
+has, `server.yaml`, the default project, and an administrator whose password is
+generated and printed once — then installs the systemd unit, starts it and
+waits until it answers. What used to be two steps of install.md is now this.
+
+Each step is skipped when its result exists, so running it again finishes an
+install that stopped and changes nothing on one that did not. `--host` adds an
+address the machine cannot see of itself (a DNS alias, a NAT or floating
+address) and reissues the certificate keeping the old ones. It never overwrites
+`server.yaml` or an authority, and certificates issued by an authority kept on
+another machine are left as they are.
+
+### Fixes
+
+- **The configuration install.md showed could not start.** An empty catalog
+  without `control.enabled: true` is refused as having nothing to serve. The
+  documentation and the file `init` writes both set it.
+
+## v0.8.4 — 2026-09-28
+
+### Node logs on the admin page
+
+Each node on the cluster page links to its logs — the agent's own, and the
+output of every engine running there — with an option to follow them. `llmgw
+ctl logs NODE` reads the same. They come over the agent's existing stream, so
+no ssh, jump host or root is needed to see what a node did. The agent keeps its
+last 2000 lines in memory; each engine its last few hundred.
+
+## v0.8.3 — 2026-09-28
+
+Found by building a node from nothing with v0.8.2.
+
+### Fixes
+
+- **A new node's first engine install failed on apt's lock.** cloud-init and
+  unattended-upgrades run apt in a machine's first minutes, and
+  `DPkg::Lock::Timeout` does not cover the lock `apt-get update` takes. A
+  package command held up by a lock now waits for up to ten minutes, and says
+  so; a failed install is retried after a minute rather than ten.
+- **A new node reported no disk** until its first download created the weights
+  directory. It is created when the agent starts.
+- **Onboarding took one control plane address.** It takes several, comma
+  separated, so a new node fails over like the rest.
+
+## v0.8.2 — 2026-09-28
+
+A node no longer runs whatever engine somebody once put on it. It installs its
+own.
+
+### Engines are installed by the node
+
+`engines:` in server.yaml names the engines every node has and their versions,
+defaulting to the llama.cpp build this release was checked against (b10991).
+Each agent converges on it: it picks the build for its hardware — CUDA with its
+runtime when an NVIDIA driver is loaded, CPU otherwise — installs the OS
+packages that build needs with the machine's own package manager, downloads the
+archive and checks it against the sha256 GitHub publishes, unpacks it refusing
+anything that would land outside its directory, and runs it. `ldd` names a
+missing library or a system too old for the build ("libstdc++.so.6 is too old
+(needs GLIBCXX_3.4.30)"); `--version` proves the rest. Installs are checked
+again every ten minutes.
+
+An engine is reported only once installed and seen to run, or when the agent's
+configuration names it on purpose. A node that has installed nothing yet is not
+a candidate for any model, where before it was a candidate for all of them.
+`ctl nodes`, the cluster page and placement's refusals say where each
+installation stands, and warn about a CPU without AVX2.
+
+Not every system is refused for its name: the build runs on Ubuntu 22.04+,
+Debian 12+ and RHEL-family 10, and `ldd` decides. vLLM and SGLang are not yet
+installed this way and must still be named in the agent's configuration.
+
+### Onboarding
+
+Onboarding installs llmgw from the CI release for the node's architecture,
+checked against the release's checksums, and writes an agent configuration of
+only the control plane's addresses and the TLS files.
+
+### Fixes
+
+- A race on the weights fingerprint, which two heartbeat loops touched during a
+  failover.
+
+## v0.8.1 — 2026-09-28
+
+Nothing here changes a configuration format: every file that worked with
+v0.8.0 still works. What changes is how much of it needs writing, and how much
+of running the cluster can be done from a page rather than a shell.
+
+### A node describes itself
+
+An agent file used to name its node, its address, the path to llama-server and
+its weight directories — different on every node and kept true by hand. The
+agent now finds these out: its id is the hostname, its address is the one on
+the route to the control plane, llama-server is looked for on PATH and in
+`/opt/llama.cpp`, a node without nvidia-smi simply has no NVIDIA cards, and
+fetched weights land in `/var/lib/llmgw-weights`, which is always scanned.
+Memory is re-read on every heartbeat and installed engines once a minute, so a
+VM given more RAM or a node given an engine becomes a candidate without an
+agent restart; the control plane logs both. The agent file shrinks to the
+control plane's addresses and the TLS files, the same on every node.
+
+### Models from the mirror, without writing YAML
+
+Search the registry from `/admin/mirror`, fetch a sharded model in one action,
+and serve a mirrored version from its row — name, source, engine and an
+estimated memory requirement filled in. A model can say what it is for, and
+`/v1/models` shows it. Placement no longer hands the same memory to two models
+or puts an engine on a node that lacks it, and says when a replica runs an
+edited profile.
+
+### Tenancy, quotas and usage on pages
+
+The tenancy tree, quotas and budgets are managed from the admin pages; an
+organization's budget bounds its teams, a group's quota counts against the
+group, and a budget that cannot be spent against says so. `/admin/usage`
+charts requests, tokens and spend over time by organization, team, project or
+person, and a request is priced by the model that served it.
+
+### Watching the cluster
+
+The cluster page has its own charts, backed by a year of Prometheus history,
+with alerts that were made to fire before being trusted — including for the
+disk the metrics share with the models.
+
+### Chat
+
+The picker offers each model once rather than once per alias, never asks for
+more tokens than the model allows, and offers a model that loads on demand —
+sending it a message is what loads it.
+
+## v0.8.0 — 2026-09-27
+
+The store stops being a file on one machine's disk. PostgreSQL, a way to move
+an existing installation onto it without reissuing every key, and the two
+things a second control plane needs so that running one is safe rather than
+clever.
+
+### A second control plane, and a way to fail over to it
+
+Leadership is a lease in the store, so two control planes against one database
+cannot both place work — which they otherwise would, both seeing a model
+wanting replicas and none running, leaving twice the replicas each holding GPUs
+the other believes are free.
+
+A standby also refuses agent streams. Commands travel down the stream the agent
+opened, so a standby holding one is a node the leader cannot command, and the
+leader's placements would fail with "no agent connected" for a node that is
+plainly connected. Agents therefore take a list of addresses rather than one,
+trying each in turn: a standby refusing, a stopped control plane refusing the
+connection and a crashed one never answering are the same thing from a node.
+
+Measured on the cluster. Leadership moved three seconds after the leader was
+stopped — it released the lease on the way out rather than letting it lapse —
+and all five agents had registered with the standby six seconds later. The
+standby adopted every replica rather than restarting any: six replicas, all
+ready, engines that never stopped serving.
+
+Everything is unset-means-leader, so a single control plane and every
+deployment that existed before this behave exactly as they did.
+
+What this does not do is move client traffic. The standby served inference
+correctly on its own port while the public address returned 502, because
+nothing in front of them moved — that needs a load balancer or a shared
+address, and it is the remaining piece.
+
+### PostgreSQL
+
+The store runs on PostgreSQL, chosen by what the address looks like: a path is
+still SQLite, a `postgres://` URL is not, and `@` followed by a path reads the
+address out of a file so a DSN carrying a password stays out of a configuration
+that gets copied and pasted into issues. Every existing installation keeps
+working untouched.
+
+One set of queries rather than one per database, because the SQL here is almost
+entirely portable and two copies would drift one query at a time. The rewrite
+of placeholders is a wrapper around the handle rather than a change at sixty
+call sites — which found six places reaching past it, including all three
+manual transactions and two prepared statements, every one of which worked on
+SQLite and would have failed on PostgreSQL.
+
+Exactly one query is not the same sentence in both: the note search, MATCH
+against `@@`. SQLite gets full-text search from an FTS5 table kept in step by
+three triggers; PostgreSQL derives a tsvector from the row, so there is nothing
+to keep in step and no trigger to forget. It uses the 'simple' configuration
+rather than English — these are an agent's own notes, in whatever language it
+chose, and stemming English rules over Turkish would quietly stop finding
+things.
+
+`llmgw keys move-store` copies an existing store rather than starting a fresh
+one, because keys are stored hashed: a migration that drops them cannot
+recreate them, and every person and every script would need a new one.
+
+Testing it against a real database found three things nothing else would have.
+`schema_migrations` is created in Go rather than in a migration file, so it
+escaped the port and still said INTEGER — 64 bits in SQLite, 32 in PostgreSQL,
+and a millisecond timestamp does not fit. The copy order followed the order the
+migrations create tables in, and `users` is created after `api_keys` even
+though `api_keys` references it. And `key_claims.sealed` is emptied the moment
+a claim is used, so a claimed row holds an empty blob in a NOT NULL column — an
+empty blob scans as a typed nil slice and a real NULL as an untyped one, a
+difference pgx cannot see, so it arrived as NULL.
+
+The driver is pgx, which is pure Go: the whole binary is built with
+CGO_ENABLED=0 so that it runs on a node whose libc is older than the builder's,
+and a cgo driver would take that away from every deployment, including the ones
+staying on SQLite.
+
+## v0.7.0 — 2026-09-25
+
+What a cluster needs to be lived with rather than demonstrated: somewhere to
+keep a conversation, somewhere to keep a note, and an honest answer about the
+disk underneath both.
+
+### Memory notes for agents, and threads without API keys
+
+Notes an agent writes for itself and reads back in later conversations, offered
+as REST (`/v1/memory`) and as MCP tools at `/mcp` (`remember`, `recall`,
+`forget`). A note is a key, text and tags, searched by words or tag, private to
+a project and to the agent id in `x-llmgw-agent`. llmgw does not inject the
+tools into chat requests: an agent that is not given the server never sees
+memory.
+
+Off for every project until someone runs `llmgw keys memory --retain-hours N`,
+and the retention is required, as it is for threads.
+
+On a gateway with no authentication, a `threads:` block (`store`,
+`retain_hours`) now keeps conversations and notes in one shared pool. Before,
+`x-llmgw-thread` was silently ignored there; now a gateway configured to keep
+nothing refuses it with an explanation instead. The shared pool is for one
+trusted machine: anyone who can reach the port and knows an id can read it.
+
+### Conversations that can move between models
+
+A thread a client names with `x-llmgw-thread`, carrying only its new turn.
+llmgw replays the stored history to whichever model is serving, which is what
+lets a conversation begun on a large model continue on a small one — including
+when a spent budget moved it there and the caller never chose.
+
+Off for every project until someone runs `llmgw keys threads`. Until then llmgw
+stores no message content anywhere, as it never has, and that default is worth
+more than the feature. The retention is required rather than optional, because
+a conversation store with no expiry is a transcript of everything a team ever
+typed kept forever by omission, and it is measured from the last turn so a
+conversation in progress does not expire underneath it.
+
+History that does not fit the serving model is trimmed oldest-first and the
+reply says how many turns went, in `x-llmgw-thread-trimmed`. System messages
+are never dropped — losing one silently changes what the model is for while it
+carries on answering — and neither is the turn the caller just sent.
+
+Threads are scoped to a project, and reading, writing or deleting one that
+belongs to another answers exactly as one that never existed: two answers would
+let a caller map which ids are real in projects they cannot read.
+
+Capturing the reply is opt-in and bounded, so a request without a thread
+accumulates nothing on the hottest path in the system, and one with a thread
+stops at a cap rather than growing to it.
+
+`DELETE /v1/threads/{id}` forgets a conversation, as a client rather than as an
+operator: the person who said the thing should not have to ask someone to read
+it before it can go.
+
+### Disk, and taking back only what llmgw put there
+
+Disk was the one resource nothing reported. `statfs` was already being called
+on every model directory, but only to ask whether the filesystem was a network
+one — the free space in the same struct was discarded, so the only way to find
+out why a node was filling up was to ssh to it.
+
+Nodes now report it, per filesystem rather than per configured directory: free
+space belongs to the disk, and two model directories on one disk are one pool
+of bytes. Shared filesystems are kept out of a node's own figure, because space
+on a filer belongs to everyone mounting it and twenty nodes would otherwise
+each report the same free terabyte. It shows in `ctl nodes`, the dashboard and
+Prometheus, and a node that reported nothing reads as unknown everywhere — an
+older agent must not look like a full disk.
+
+A profile may declare `min_disk_gb`, and a node without the room is no longer a
+candidate, with a reason `explain-placement` prints. Skipped on a node that
+already holds the weights: it needs no room for them, and without that the node
+best placed to run a model is the one most likely to be rejected.
+
+`ctl weights` lists model files llmgw downloaded for models no longer in the
+catalog, and `--collect` deletes them. The design is mostly about what it will
+not do. Only files llmgw fetched are candidates, and that is a calculation
+rather than a heuristic — the cache name is a hash of the source, so the wanted
+set is computable exactly. Anything an operator put on a node is left alone
+however unreferenced it looks. Each node checks every path again before
+unlinking it, refusing anything outside its cache directory or without the
+cache name, so a control plane that is wrong cannot talk a node into losing a
+model that took a week to copy.
+
+### Fixes
+
+- **A capability was lost at an indirection.** The controller is handed a
+  forwarding dispatcher, and reaches past its interface by type assertion to
+  find what lies beyond scheduling. Deleting weights was implemented on the
+  agent service and not on the forwarder, so it was simply absent at runtime —
+  with the whole suite green, because the dispatchers the tests use do not
+  implement it either and its absence there is the expected case.
+- **`ctl tool-check` blamed the engine for what the model did.** It had one
+  verdict for a failed attempt and told operators not to try another model.
+  Measured: the engine renders the tool prompt correctly, the coder models do
+  not use the `<tool_call>` wrapper it asks for, and `qwen2.5-3b-instruct` on
+  the same engine returns a populated `tool_calls` array. A failed attempt is
+  now two verdicts, told apart by whether the reply used the wrapper.
+
+## v0.6.0 — 2026-09-18
+
+Phase 4: a replica can be bigger than one machine, and the fabric it spans
+decides where it is allowed to go. Around that, everything a cluster needs
+before anyone other than its operator is let near it — keys, budgets, a way in
+through a browser, and enough visibility to answer "why is nothing scheduling".
+
+### Multi-node replicas
+
+A replica is now a set of shards with ranks, a leader and a bootstrap address,
+rather than one process. Two invariants are enforced rather than documented: a
+replica never spans two fabrics, and a tensor-parallel group never mixes GPU
+classes. Both are the kind of mistake that does not fail — it hangs, or it runs
+several times slower, and nothing says which.
+
+A replica missing a shard is not ready. It was previously derived from the
+shards that had reported, so a two-node replica with one node up looked
+servable and took traffic into a model that could not answer.
+
+Adoption after a restart happens one agent at a time, because that is how
+agents reconnect. Merging them as they arrive is what keeps a multi-node
+replica whole instead of being torn into fragments that are then killed as
+strays — and an adopted engine is health-checked before it is called ready,
+rather than being assumed alive because its process exists.
+
+### Fabrics are distribution domains too
+
+An isolated fabric is not only a compute boundary: weights present in one are
+not present in the other. Node weight inventories are reported and summarised
+per fabric, so "the model is on the cluster" is never mistaken for "the model
+is reachable from the nodes that would run it". A fabric name that matches
+nothing a node actually has is refused at registration rather than becoming an
+unschedulable replica hours later.
+
+### Who may use it
+
+API keys, tenants and per-model allowlists, on SQLite (`modernc.org/sqlite`, so
+still no cgo). Rate limits and budgets are enforced on the request path; usage
+is written off it, in batches, so accounting never adds latency to inference.
+Budgets survive a restart — counters are rebuilt from the store, because a
+limit anyone can clear by bouncing the process is not a limit.
+
+A spent budget is now served a cheaper model where the catalog offers one,
+rather than refused. For a team trying models out, being dropped to something
+smaller is a far better answer than a wall, and the response says which model
+answered.
+
+People can sign up, be approved, and issue and revoke their own keys. A key is
+shown once; only its hash is kept.
+
+### The browser surface
+
+A landing page that says how to connect, a read-only cluster dashboard, and an
+in-browser chat that never puts an API key in the browser. The dashboard shows
+who is using the cluster and what they are running — per person and per model,
+with requests, tokens, errors, spend and latency.
+
+All of it is served from the client-facing port behind a session, deliberately
+not from the admin port, which also serves pprof — and pprof can dump the heap
+that holds every API key in plaintext.
+
+### Models without a restart
+
+The catalog is live. Models can be added and removed from the command line or
+the API, take effect immediately, and outlive a restart; configuration still
+wins over the store, and the whole catalog is validated before anything is
+written.
+
+A model is only listed once it can actually serve. It was listed as soon as it
+was known, so the first thing a new user did was ask for a model that returned
+503.
+
+### Loading on demand, and letting go
+
+A model whose scaling allows it is started when someone asks for it, and
+stopped when nobody has for a while. An idle replica is evicted to make room
+for one that was asked for, with guards that exist because each is a way to
+lose work quietly: only on-demand models, only single-node replicas, never one
+that has never been used, and never one younger than its minimum lifetime.
+
+Unknown is not idle. A replica with no recorded use is left alone rather than
+treated as idle since the epoch.
+
+### Knowing whether it will run before it runs
+
+- `ctl plan` works out whether a model fits the cluster and how it would be
+  split, in the precision the chosen engine can actually serve — recommending
+  a quantisation the engine cannot load would cause the failure the command
+  exists to prevent.
+- `ctl tool-check` verifies llmgw's engine flags against the engines actually
+  installed. Flags drift between releases, and the alternative is finding out
+  during a model load.
+- Placement now knows a model's context length and memory footprint, and node
+  RAM, so a CPU profile is not placed on a machine that cannot hold it.
+- Placement waits for the fleet to settle after a restart rather than deciding
+  against a half-reported cluster.
+
+### Weights
+
+Nodes report the model files they can see, without llmgw owning the storage:
+a shared filer, an image exported over the fabric, rsync or a copy made by hand
+all look the same to placement, which only cares whether the files are there.
+Reporting nothing means "not asked to look", not "has nothing".
+
+Where the simple arrangement is not enough, a node can fetch a model by URL —
+single-flight, resumable, written to a temporary name and renamed atomically,
+so a killed download never leaves something that looks like weights.
+
+### Seeing the fleet
+
+Prometheus metrics for nodes, GPUs and replicas, collected from live state at
+scrape time rather than from a background goroutine, and served on their own
+endpoint rather than through the one that also exposes pprof. The same view is
+in the dashboard, so nothing else needs installing to see it.
+
+### Onboarding a node
+
+Adding a node is one command. There are systemd units, a private CA and a
+from-nothing install guide that starts with a machine that has no CUDA and no
+binary.
+
+### llama.cpp, and CPU profiles
+
+A third engine adapter, and the reason for it: llama.cpp serves the same OpenAI
+API without a GPU, so placement, supervision, scraping, scheduling and admission
+can all be exercised on ordinary machines.
+
+A profile may now omit `gpu_class`, which is how it says it runs on the CPU. Only
+engines that can actually serve without a card may omit it — for vLLM an omitted
+class stays an error, because otherwise a typo becomes a replica placed on a
+machine with no hardware and diagnosed minutes later during model loading. CPU
+replicas are spread across nodes rather than bin packed: unlike GPUs there is no
+allocation to stop two of them sharing a machine's cores.
+
+### Metric mappings are per engine
+
+`Running`, `Waiting` and `KVCacheUsage` now have a vocabulary per engine rather
+than one for the cluster. A mixed fleet has two names for the same three numbers,
+and a single global mapping left half of it unmeasurable.
+
+A mapping that leaves a metric unnamed now means the engine does not publish it,
+and the field stays zero. Naming one the engine does not serve is still a contract
+violation. `Running` and `Waiting` can never be omitted.
+
+### Mutual TLS on the agent stream
+
+A private CA (`llmgw ctl cert init-ca | issue-server | issue-client | inspect`),
+certificates on both ends, and the control plane refusing any agent whose claimed
+node id does not match its certificate's common name. Public authorities cannot
+supply these: they stopped issuing the `clientAuth` usage.
+
+Plaintext HTTP/2 remains the default. `agent_tls:` on the server and `tls:` on the
+agent turn it on.
+
+### Fixes
+
+- **An agent told to stop did not stop.** Cancelling the context is not enough —
+  HTTP/2 watches it while the request body is being written, and a bidirectional
+  stream spends its life blocked on the read side. Measured on a deployed node:
+  `systemctl stop` took 90 seconds and ended in SIGKILL. Now 0.01 seconds.
+- **An agent could not explain why it was refused.** The session cancelled its own
+  context and then asked that context whether it was shutting down, so every
+  stream error read as a clean close: no message, and the backoff reset.
+- **A broken metric contract was invisible.** It was logged per scrape at debug
+  level, so a cluster could look healthy while every routing decision was made
+  blind. It is now a warning, once, with a matching line when it clears.
+- **Token estimates came from the routing prefix, not the prompt.** Every prompt
+  over about 2 KB was undercounted, so tokens-per-minute limits let through far
+  more than they were set to.
+- **Model-fitting double-counted overhead.** Whole-model overhead was charged per
+  rank and a 10% reserve taken twice, so `plan` reported that a 70B model in
+  bfloat16 does not fit two 80 GB cards. It does.
+- **`x-llmgw-served-model` was never sent.** It was set after the proxy had
+  already written the response header, which is the one point at which headers
+  stop being changeable. A client had no way to tell a fallback from the model
+  it asked for.
+- **Fleet metrics published nothing.** They were registered after the registry
+  had been built, so the collector existed and was never scraped.
+- **GPU discovery broke on older drivers.** A field added for utilisation was
+  required rather than optional, so a driver that does not report it produced no
+  inventory at all instead of an inventory without that number.
+- **Replica ids and ports were reused** while the processes holding them were
+  still alive.
+
+## v0.5.0 — 2026-09-16
+
+Phases 2 and 3. The cluster runs itself: agents report their hardware, the
+control plane places replicas, and `ctl` makes all of it visible.
+
+This release covers both phases because their exit criteria depended on the same
+piece — the agent stream — and neither was demonstrable without it.
+
+### ctl
+
+```
+llmgw ctl nodes | models | replicas | explain-placement | cordon | uncordon | drain
+```
+
+- `nodes` surfaces unhealthy GPUs and why they are excluded, which is usually
+  the answer when a node looks idle but nothing schedules onto it.
+- `explain-placement` prints every candidate considered and the reason it was
+  rejected. "Unschedulable" on its own says nothing about which of six
+  constraints bit.
+- `drain` is a rolling move, not a stop: the replica is drained, a replacement
+  is placed elsewhere, and the original is stopped only once the replacement is
+  serving. Bounded by a grace period, because draining the only node that can
+  host a model would otherwise never finish and leave it undrainable.
+- `cordon` is the weaker operation: no new placement, nothing disturbed.
+
+The management API is on the admin listener, not the client-facing one. The two
+have different audiences: one serves inference to whoever holds an API key, the
+other lets a caller cordon nodes.
+
+`ctl logs` is not here. It needs a request/response pair the agent protocol does
+not have. The failure path already ships an engine log tail with every status
+report, so the case that matters most is covered; reading a healthy shard's
+output on demand is not.
+
+### Phases 2 and 3, in detail
+
+Written as the work landed, which is why it reads as a log rather than as
+notes. Everything here shipped in v0.5.0.
+
+### Added
+
+- **IB discovery** (`internal/agent/discovery`): reads `/sys/class/infiniband`,
+  parses the formats sysfs actually uses (`"4: ACTIVE"`, `"200 Gb/sec (4X HDR)"`,
+  hex or decimal LIDs), and distinguishes native InfiniBand from RoCE. A node
+  without IB is not an error — it is a single-node island.
+- **GPU discovery** via `nvidia-smi`, the cgo-free path. NVML needs cgo, which
+  would force the whole binary, server included, to be built with a C
+  toolchain. Health flags anything that should keep a card out of placement:
+  uncorrected ECC errors, and memory already held by another process.
+- **GPU class derivation.** Sizes come from the driver in GiB, which does not
+  always match the marketing name: an L40S is sold as 48GB and reports 44.99
+  GiB, so its class is `l40s-45g`. Consistency matters more than matching the
+  box, because the number decides whether a model fits.
+- **Engine adapters** for vLLM and SGLang: command and environment
+  construction, kept free of process handling so it is testable without GPUs.
+  vLLM supports both Ray and multiprocessing launch modes, because which one
+  works depends on the version.
+- **NCCL environment generation**, the part plan §5.3 marks critical.
+  `NCCL_IB_HCA` lists only the chosen fabric's adapters — leaving it unset lets
+  NCCL pick an adapter on another fabric where the peer is unreachable, and the
+  job hangs rather than failing. Bootstrap is pinned to the management
+  interface for the same reason.
+- **Validation before launch**: a tensor-parallel group may not mix GPU types,
+  the node/GPU/TP arithmetic has to add up, and an FP8 profile is refused below
+  compute capability 8.9 rather than silently falling back to a slower path.
+- **Supervisor** (`internal/agent/supervisor`): processes run in their own
+  process group and are stopped as a group, because engines fork workers that
+  hold GPU memory and outlive a signal sent only to the leader. Bounded log
+  tail, a state machine that separates Starting from Loading (a 70B model takes
+  minutes, and an operator needs to know that is normal), and orphan adoption
+  through pid files fingerprinted with the command line — pids get reused, and
+  adopting an unrelated process is worse than leaking the original.
+- **Readiness** is two questions: `/health` says the process is up, `/v1/models`
+  says the weights loaded. Routing on `/health` alone sends traffic into a model
+  that is still loading.
+- **`llmgw agent inventory`** reports what the scheduler would see on a node,
+  as a table or JSON. Run it before adding a node: it is the fastest way to find
+  a card another process is using, or a port with no subnet manager.
+
+- **NVML backend** behind the `nvml` build tag, which adds NVLink topology.
+  Nothing else exposes which GPUs share an NVLink domain, and that is what keeps
+  a tensor-parallel group off the slow path inside a node. `Discover` falls back
+  to nvidia-smi if NVML fails, and says so loudly rather than silently losing the
+  topology. CI compiles the tagged build so it cannot rot unnoticed.
+
+### Not verified
+
+The NVML path compiles but has never run against a driver — there is no GPU
+machine to try it on. Treat the NVLink domains it reports as unverified until
+someone runs `llmgw agent inventory` on a real node.
+
+- **Control plane wiring** (`internal/control`): Ready replicas become router
+  endpoints, with no static configuration involved. The router keeps live state
+  for endpoints that survive an update — inflight counts, breaker state, drain
+  rate — so a replica that was already serving does not start from cold because
+  another one moved. The sink is an interface, so when the router and control
+  plane split into separate processes this becomes a watch stream and nothing
+  above it changes.
+
+### Phase 3 exit criterion, measured in `test/e2e`
+
+A node hosting a live replica is killed while four clients stream continuously.
+The control plane marks it unreachable, then lost, rebuilds the replica on a
+survivor, and republishes. Across three runs: **~370 requests completed, zero
+failures**, and no replica remained on the dead node.
+
+This proves the control logic, not the transport: the agent fleet is simulated,
+and the real path still needs the stream.
+
+### A real bug the race detector caught
+
+`startReplica` read the replica's fields after publishing it to the map, and
+`scaleUp` walked replica pointers outside the lock. Both were racing with agent
+status reports, which arrive from another goroutine and mutate exactly those
+fields. Under the simulated fleet the reports came back fast enough to hit the
+window every time. Everything is snapshotted under the lock now and the
+functions work from values.
+
+### The agent stream
+
+- **Protocol** (`api/proto/llmgw/v1`): one bidirectional connect-go stream
+  carrying registration, heartbeats and shard status upward, and start, stop and
+  drain commands downward. The agent dials out, because GPU nodes sit behind
+  firewalls far more often than a control plane does and an outbound connection
+  needs no inbound rule on every node in the fleet.
+- **Registration carries the whole inventory**, so there is no separate
+  discovery call and a reconnecting agent refreshes what the scheduler knows.
+  The fabric labels travel with it — they cannot be inferred from the wire, so
+  losing them in transit would silently disable multi-node placement.
+- **Agent service** (`internal/control/agentsvc`) is both the stream handler and
+  the reconciler's dispatcher: it holds the open streams, so it is the only
+  thing that can deliver a command to a node.
+- **Agent** (`internal/agent`) validates a profile before launching anything —
+  an engine that refuses a profile does so after minutes of loading — then
+  supervises the process and reports every transition, including the engine's
+  output tail on failure.
+- **Reconnects forever with jittered backoff.** A control plane restart must not
+  require touching every GPU node, and a fleet must not reconnect in lockstep
+  and knock the control plane over again. Engines keep serving throughout: the
+  router talks to them directly.
+- **`llmgw agent` is real**, and `llmgw server` runs the control plane whenever
+  any model declares a profile. The static endpoint list remains for
+  router-only runs against sim-engines.
+
+### Verified with real binaries
+
+Server and agent as separate processes over a real stream: the agent registers,
+a replica is placed, an engine is launched through the vLLM adapter's actual
+argument construction, streaming traffic is served through the router, and
+killing the agent moves the node through unreachable to lost with placement
+explaining exactly why it can no longer run anything.
+
+### Surviving a control plane restart
+
+Not persisting replica state left one hole that had to be closed before it
+mattered: restarting the control plane orphaned live engines. They kept holding
+GPUs the inventory believed were free, so the next placement failed to allocate
+with an error naming the new process rather than the old one.
+
+The fix is on the agent, not in a database, and it cannot disagree with reality
+the way a stale row can:
+
+- The supervisor now writes the full shard spec, not just a pid and command
+  line. A pid identifies a process; the spec is what lets it be described back
+  to the control plane and matched to a replica.
+- `Adopt` takes over a still-running engine from a previous agent. It cannot be
+  waited on — it is not this process's child — so it is watched by polling, and
+  stopped by signalling its group directly. Its earlier output is gone, and the
+  log says so rather than presenting emptiness as silence.
+- Every `Register`, not only the first, reports what the agent is running. A
+  reconnect may be a control plane restart and has to recover the same way.
+- The reconciler rebuilds replicas from those reports and reclaims their GPUs.
+  Anything whose model or profile left the catalog is stopped: nothing will
+  route to it and it is holding cards.
+- An engine the agent cannot identify — a corrupt state file — is killed. A
+  process holding GPUs that nothing can describe is worse than no process,
+  because placement will hand those GPUs out again.
+
+One mechanism covers agent restarts and control plane restarts.
+
+### A test that was passing for no reason
+
+`TestAgentReconnects` used `httptest.CloseClientConnections`, which does nothing
+to an h2c stream: prior-knowledge h2c hijacks the connection, and httptest stops
+tracking hijacked connections. The agent never reconnected and the assertion —
+on the connection map, which still held a stale entry — passed anyway. Tests now
+sever connections through their own listener and count opened streams.
+
+### Multi-node launch
+
+Placement and environment generation had landed early, which made multi-node
+look further along than it was: a multi-node replica could not come up at all.
+Three things were missing, and all three came down to two assumptions that do
+not hold once a replica spans nodes.
+
+**Not every rank serves HTTP.** The agent health-checked every shard at its own
+port, but only the leader answers. Workers timed out, reported Failed, and took
+the replica with them. `Engine.ServesHTTP(spec)` now says which ranks serve; a
+worker is judged by staying alive instead. That is weaker evidence on purpose —
+the replica's real readiness is the leader answering, which only happens once
+every worker has joined.
+
+**A shard is not always one process.** `Engine.Commands(spec)` returns a
+sequence: every command but the last is a sidecar kept alive for the shard's
+lifetime and torn down with it, and the last one's exit is the shard's exit.
+vLLM under Ray is why — the leader runs a Ray head and then the server against
+it, and a worker runs only the join, with no vLLM process at all. `RayCommand`
+had been written and tested but never called, so `launch_mode: ray` would have
+run `vllm serve` against a cluster that did not exist.
+
+A command can also require the rendezvous point to be listening before the next
+one starts, bounded by a timeout so a head that never appears fails the shard
+rather than hanging it. Sidecars are killed with the shard: a Ray head left
+behind holds the port the next replica needs.
+
+**Start order was backwards.** Workers were started first, on the theory that
+the head should wait for them. Both engines put the rendezvous point on rank 0,
+so a worker started first has nothing to connect to and spends its startup
+budget retrying. vLLM's server blocks until the Ray cluster has the GPUs it
+needs, so the leader can start immediately and waits on its own.
+
+None of this is verified against real engines. It is tested with stub processes;
+the Ray sequencing and which launch mode a given vLLM version wants can only be
+confirmed on hardware.
+
+### Still missing for phase 2
+
+- mTLS. The stream is plaintext HTTP/2 today, so it must stay on a trusted
+  network.
+
+### Phase 3: placement
+
+- **Cluster inventory** (`internal/control/inventory`): nodes, their hardware,
+  their fabric attachments, and what is allocated where. A re-registering agent
+  cannot clear an operator's cordon, and a heartbeat revives an unreachable node
+  without doing so either. A GPU with an ECC fault or another process on it is
+  not counted as free. Replicas are released whole, because they fail whole.
+- **Placement** (`internal/control/placement`): picks the highest-priority
+  profile that fits and explains every candidate it rejected. Two constraints
+  are enforced structurally rather than checked afterwards — there is no code
+  path that can combine nodes from two fabrics, and only GPUs of the profile's
+  class are ever considered.
+- **Bin packing** fills partly used nodes first, so whole nodes stay free for
+  large models that cannot be split. Multi-node replicas get the opposite
+  treatment: they are the least movable thing in the cluster, so they take the
+  emptiest machines.
+- **NVLink-aware GPU selection** keeps a tensor-parallel group inside one NVLink
+  domain where the topology is known. Without the NVML build it falls back to
+  the lowest free indices, which is usually right but only by accident of how
+  vendors number slots.
+- **Property-based invariant tests** (plan §13): 400 randomly generated
+  clusters, checking on every placement that a multi-node replica lives on one
+  fabric, that each shard receives only that fabric's adapters, and that no
+  placement mixes GPU classes or uses an unhealthy or already-allocated GPU.
+  The table-driven tests cover the cases someone thought of; this covers the
+  ones nobody did.
+- **Catalog** (`internal/control/catalog`): models, profiles, aliases,
+  fallbacks and scaling bounds. Aliases and fallbacks are resolved at load
+  rather than at request time — a typo in a fallback should stop a deployment,
+  not surface during the outage when the fallback is finally needed. The
+  node/GPU/TP arithmetic is checked here too, so a bad profile fails at config
+  load instead of minutes into model loading.
+- **Reconciler** (`internal/control/reconciler`): the desired-versus-actual
+  loop. It places one replica per pass, because placement depends on what is
+  already allocated and starting several from one snapshot would hand them the
+  same GPUs. Workers start before rank 0, since SGLang's ranks rendezvous and
+  vLLM under Ray needs its workers attached before the head serves.
+- **Replica failure is whole-replica failure.** One dead shard makes the others
+  useless — they wait on collectives that will never complete while holding
+  GPUs — so the reconciler stops every shard and rebuilds, with a jittered
+  exponential backoff carried on the model rather than the replica it just
+  destroyed.
+- **Node expiry in two stages.** Unreachable first, Lost later. A brief network
+  blip must not tear down replicas that are still serving, and rebuilding a
+  multi-node replica costs tens of minutes of model loading.
+- **Scale-down drains before stopping**, so in-flight requests finish instead of
+  being cut off mid-stream.
+
 ## v0.3.0 — 2026-09-16
 
 Flow control. A batch workload no longer starves interactive traffic.

Makefile

@@ -2,7 +2,7 @@ MODULE  := gitea.mynodes.xyz/ilker/llmgw
 VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
 LDFLAGS := -X '$(MODULE)/internal/version.Version=$(VERSION)'
 
-.PHONY: build test race lint fmt clean rename
+.PHONY: build test race lint fmt clean rename proto proto-tools
 
 build:
 	go build -ldflags "$(LDFLAGS)" -o bin/llmgw ./cmd/llmgw
@@ -20,6 +20,17 @@ fmt:
 	go fmt ./...
 	go vet ./...
 
+# Regenerate the agent protocol. Generated code is committed, so this only
+# needs running when api/proto changes.
+proto: proto-tools
+	buf lint
+	buf generate
+
+proto-tools:
+	go install github.com/bufbuild/buf/cmd/buf@latest
+	go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
+	go install connectrpc.com/connect/cmd/protoc-gen-connect-go@latest
+
 clean:
 	rm -rf bin dist
 

README.md

@@ -6,6 +6,18 @@ heterogeneous GPUs.
 It serves LLMs running on your own GPU servers (vLLM / SGLang) behind one
 OpenAI-compatible endpoint. Design document: [llmgw-plan.md](llmgw-plan.md).
 
+## Getting started
+
+Starting from nothing — no packages installed, no drivers, no nodes — follow
+[docs/install.md](docs/install.md): it builds a control plane, adds a node and
+serves a model, and says plainly which parts llmgw does not install for you.
+
+| Guide | For |
+|---|---|
+| [docs/install.md](docs/install.md) | Standing up a cluster from scratch |
+| [docs/node-onboarding.md](docs/node-onboarding.md) | Adding nodes to one that exists |
+| [docs/onboarding.md](docs/onboarding.md) | Giving people accounts and keys |
+
 ## Status
 
 Releases track the roadmap phases; see [CHANGELOG.md](CHANGELOG.md).
@@ -15,15 +27,49 @@ Releases track the roadmap phases; see [CHANGELOG.md](CHANGELOG.md).
 | v0.1.0 | 0 | Skeleton and sim-engine | released |
 | v0.2.0 | 1 | Static router (filter/score/pick, streaming proxy) | released |
 | v0.3.0 | 1.5 | Admission control (priority bands) | released |
-| v0.4.0 | 2 | Agent: GPU/IB discovery, single-node engines | planned |
-| v0.5.0 | 3 | Control plane: catalog, scheduler, reconciler | planned |
-| v0.6.0 | 4 | Multi-node placement and fabric awareness | planned |
+| v0.4.0 | 2 | Agent: GPU/IB discovery, engine supervision | released |
+| v0.5.0 | 3 | Control plane: catalog, placement, reconciler, ctl | released |
+| v0.6.0 | 4 | Multi-node placement and fabric awareness | released |
+| v0.7.0 | — | Conversations, agent memory, disk management | released |
+| v0.8.0 | — | PostgreSQL, leader election, agent failover | released |
 | v1.0.0 | 5 | Tenancy, prefix affinity, benchmarking | planned |
 
-As of v0.3.0 the router serves traffic and protects it: a batch workload no
-longer starves interactive requests. Still missing before v1.0.0: authentication,
-rate limits and budgets, and the control plane that replaces the static endpoint
-list.
+As of v0.6.0 a cluster runs unattended: agents report their hardware, the
+control plane places replicas — including ones larger than a single machine —
+models load when someone asks for them and stop when nobody does, and keys,
+budgets and rate limits decide who may use any of it.
+
+Before v1.0.0: tracing, reference documentation, and filtering on `ctl nodes`
+and `ctl replicas` for fleets too large to read a whole table of. Two items
+that were on this list are not gaps: a model falls through to its fallbacks
+when there is no capacity as well as when a budget is spent, and routing
+profiles are chosen by the key rather than by a request header on purpose —
+a header would let any client promote itself into the top priority band.
+
+One thing is worth saying plainly:
+multi-node replicas are proven on a simulated cluster and on real hardware
+without a fabric. The InfiniBand, NVML and NCCL paths are written against the
+formats and failure modes those tools actually have, but they have not yet run
+on a machine that has them.
+
+## Documentation
+
+- [docs/install.md](docs/install.md) — installing from nothing: control plane,
+  nodes, engines, weights, first request
+- [docs/quickstart.md](docs/quickstart.md) — the short path
+- [docs/choosing-models.md](docs/choosing-models.md) — which models to run,
+  how many of each, and where the weights should live
+- [docs/data.md](docs/data.md) — what llmgw stores, what it does not, and
+  what leaves the machine; written for a security review
+- [docs/reference.md](docs/reference.md) — every configuration key, generated
+  from the structs and checked by a test, so it cannot describe a setting that
+  does not exist or miss one that does
+- [deploy/mirror/](deploy/mirror/) — a model mirror, which is the cheapest
+  and most useful thing to build around llmgw
+- [deploy/ha/](deploy/ha/) — running two control planes, and the one health
+  check that has to be right
+- [docs/node-onboarding.md](docs/node-onboarding.md),
+  [docs/onboarding.md](docs/onboarding.md) — adding machines, and adding people
 
 ## Building
 
@@ -33,6 +79,24 @@ make test       # unit tests
 make race       # with the race detector
 ```
 
+## Operating a cluster
+
+```sh
+llmgw ctl nodes                      # capacity, health, what is running where
+llmgw ctl models                     # catalog and replica counts
+llmgw ctl replicas --model llama     # replicas and their shards
+llmgw ctl explain-placement llama    # why a model has not been placed
+llmgw ctl drain gpu-a-01             # move work off a node, one replica at a time
+llmgw ctl cordon gpu-a-01            # stop new placement, leave what runs
+```
+
+`ctl` talks to the admin port (`:9090` by default), not the client-facing one.
+Add `--json` to any command for machine-readable output.
+
+`explain-placement` is the one worth knowing about: "unschedulable" on its own
+says nothing about which of six constraints bit, so it prints every candidate
+considered and the reason it was rejected.
+
 ## Running the router
 
 ```sh

api/gen/llmgw/v1/llmgwv1connect/agent.connect.go

@@ -0,0 +1,119 @@
+// Code generated by protoc-gen-connect-go. DO NOT EDIT.
+//
+// Source: llmgw/v1/agent.proto
+
+// Package llmgw.v1 is the protocol between a node agent and the control plane
+// (plan §5.6).
+//
+// The agent dials out and holds a bidirectional stream open. That direction is
+// deliberate: GPU nodes sit behind firewalls and NAT far more often than a
+// control plane does, and an outbound connection needs no inbound rule on every
+// node in the fleet.
+package llmgwv1connect
+
+import (
+	connect "connectrpc.com/connect"
+	context "context"
+	errors "errors"
+	v1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+	http "net/http"
+	strings "strings"
+)
+
+// This is a compile-time assertion to ensure that this generated file and the connect package are
+// compatible. If you get a compiler error that this constant is not defined, this code was
+// generated with a version of connect newer than the one compiled into your binary. You can fix the
+// problem by either regenerating this code with an older version of connect or updating the connect
+// version compiled into your binary.
+const _ = connect.IsAtLeastVersion1_13_0
+
+const (
+	// AgentServiceName is the fully-qualified name of the AgentService service.
+	AgentServiceName = "llmgw.v1.AgentService"
+)
+
+// These constants are the fully-qualified names of the RPCs defined in this package. They're
+// exposed at runtime as Spec.Procedure and as the final two segments of the HTTP route.
+//
+// Note that these are different from the fully-qualified method names used by
+// google.golang.org/protobuf/reflect/protoreflect. To convert from these constants to
+// reflection-formatted method names, remove the leading slash and convert the remaining slash to a
+// period.
+const (
+	// AgentServiceConnectProcedure is the fully-qualified name of the AgentService's Connect RPC.
+	AgentServiceConnectProcedure = "/llmgw.v1.AgentService/Connect"
+)
+
+// AgentServiceClient is a client for the llmgw.v1.AgentService service.
+type AgentServiceClient interface {
+	// Connect opens the agent's stream. It stays open for the agent's lifetime;
+	// losing it is how each side learns the other is gone.
+	Connect(context.Context) *connect.BidiStreamForClient[v1.AgentMessage, v1.ControlMessage]
+}
+
+// NewAgentServiceClient constructs a client for the llmgw.v1.AgentService service. By default, it
+// uses the Connect protocol with the binary Protobuf Codec, asks for gzipped responses, and sends
+// uncompressed requests. To use the gRPC or gRPC-Web protocols, supply the connect.WithGRPC() or
+// connect.WithGRPCWeb() options.
+//
+// The URL supplied here should be the base URL for the Connect or gRPC server (for example,
+// http://api.acme.com or https://acme.com/grpc).
+func NewAgentServiceClient(httpClient connect.HTTPClient, baseURL string, opts ...connect.ClientOption) AgentServiceClient {
+	baseURL = strings.TrimRight(baseURL, "/")
+	agentServiceMethods := v1.File_llmgw_v1_agent_proto.Services().ByName("AgentService").Methods()
+	return &agentServiceClient{
+		connect: connect.NewClient[v1.AgentMessage, v1.ControlMessage](
+			httpClient,
+			baseURL+AgentServiceConnectProcedure,
+			connect.WithSchema(agentServiceMethods.ByName("Connect")),
+			connect.WithClientOptions(opts...),
+		),
+	}
+}
+
+// agentServiceClient implements AgentServiceClient.
+type agentServiceClient struct {
+	connect *connect.Client[v1.AgentMessage, v1.ControlMessage]
+}
+
+// Connect calls llmgw.v1.AgentService.Connect.
+func (c *agentServiceClient) Connect(ctx context.Context) *connect.BidiStreamForClient[v1.AgentMessage, v1.ControlMessage] {
+	return c.connect.CallBidiStream(ctx)
+}
+
+// AgentServiceHandler is an implementation of the llmgw.v1.AgentService service.
+type AgentServiceHandler interface {
+	// Connect opens the agent's stream. It stays open for the agent's lifetime;
+	// losing it is how each side learns the other is gone.
+	Connect(context.Context, *connect.BidiStream[v1.AgentMessage, v1.ControlMessage]) error
+}
+
+// NewAgentServiceHandler builds an HTTP handler from the service implementation. It returns the
+// path on which to mount the handler and the handler itself.
+//
+// By default, handlers support the Connect, gRPC, and gRPC-Web protocols with the binary Protobuf
+// and JSON codecs. They also support gzip compression.
+func NewAgentServiceHandler(svc AgentServiceHandler, opts ...connect.HandlerOption) (string, http.Handler) {
+	agentServiceMethods := v1.File_llmgw_v1_agent_proto.Services().ByName("AgentService").Methods()
+	agentServiceConnectHandler := connect.NewBidiStreamHandler(
+		AgentServiceConnectProcedure,
+		svc.Connect,
+		connect.WithSchema(agentServiceMethods.ByName("Connect")),
+		connect.WithHandlerOptions(opts...),
+	)
+	return "/llmgw.v1.AgentService/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case AgentServiceConnectProcedure:
+			agentServiceConnectHandler.ServeHTTP(w, r)
+		default:
+			http.NotFound(w, r)
+		}
+	})
+}
+
+// UnimplementedAgentServiceHandler returns CodeUnimplemented from all methods.
+type UnimplementedAgentServiceHandler struct{}
+
+func (UnimplementedAgentServiceHandler) Connect(context.Context, *connect.BidiStream[v1.AgentMessage, v1.ControlMessage]) error {
+	return connect.NewError(connect.CodeUnimplemented, errors.New("llmgw.v1.AgentService.Connect is not implemented"))
+}

api/proto/llmgw/v1/agent.proto

@@ -0,0 +1,448 @@
+syntax = "proto3";
+
+// Package llmgw.v1 is the protocol between a node agent and the control plane
+// (plan §5.6).
+//
+// The agent dials out and holds a bidirectional stream open. That direction is
+// deliberate: GPU nodes sit behind firewalls and NAT far more often than a
+// control plane does, and an outbound connection needs no inbound rule on every
+// node in the fleet.
+package llmgw.v1;
+
+option go_package = "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1;llmgwv1";
+
+service AgentService {
+  // Connect opens the agent's stream. It stays open for the agent's lifetime;
+  // losing it is how each side learns the other is gone.
+  rpc Connect(stream AgentMessage) returns (stream ControlMessage);
+}
+
+// AgentMessage is anything a node tells the control plane.
+message AgentMessage {
+  oneof msg {
+    Register register = 1;
+    Heartbeat heartbeat = 2;
+    ShardStatus shard = 3;
+    LogsReply logs = 4;
+  }
+}
+
+// ControlMessage is anything the control plane tells a node.
+message ControlMessage {
+  oneof msg {
+    StartShard start = 1;
+    StopShard stop = 2;
+    DrainShard drain = 3;
+    Ack ack = 4;
+    RemoveWeights remove_weights = 5;
+    Provision provision = 6;
+    LogsRequest logs = 7;
+  }
+}
+
+// Register is the first message on every stream. It carries the node's full
+// hardware inventory, so the control plane needs no separate discovery call and
+// a reconnecting agent automatically refreshes what the scheduler knows.
+message Register {
+  string node_id = 1;
+  string hostname = 2;
+  // mgmt_addr is the address engines are reachable on, and the address NCCL
+  // bootstraps over. Never a fabric address: NCCL rendezvouses over TCP before
+  // it moves to RDMA.
+  string mgmt_addr = 3;
+  map<string, string> labels = 4;
+
+  repeated GPU gpus = 5;
+  // fabrics is empty on a node without InfiniBand. Such a node is its own
+  // single-node island and can never host a multi-node replica.
+  repeated FabricAttachment fabrics = 6;
+
+  string agent_version = 7;
+  // engine_versions decides which CLI flags and metric names apply; both drift
+  // between engine releases (plan §14).
+  map<string, string> engine_versions = 8;
+
+  // running_shards are the engines this agent is already supervising, including
+  // ones adopted from a previous agent.
+  //
+  // This is what lets a control plane restart recover instead of orphaning live
+  // engines. Without it the control plane forgets every replica, places new
+  // ones, and the old engines keep holding GPUs the inventory believes are free
+  // — so the new engines fail to allocate, with an error naming the wrong
+  // process (plan §6.5).
+  repeated RunningShard running_shards = 9;
+
+  // memory_total_mb is the machine's RAM, and 0 means the agent did not say —
+  // which is what every agent older than this field reports, and is treated as
+  // no constraint rather than as a node with no memory.
+  //
+  // Placement needs it because a CPU profile has no card to be free, so every
+  // ready node looks like a candidate: a 14B model whose working set is 24 GB
+  // would be placed just as happily on a node with 8 GB.
+  int64 memory_total_mb = 10;
+
+  // weights are the model files this node can see, from the directories its
+  // configuration names.
+  //
+  // How they got there is the operator's business and deliberately outside
+  // llmgw's: a shared filesystem, an image exported over the fabric, rsync, or
+  // a copy made by hand all look identical here. What matters to placement is
+  // only whether they are present, because a node without them either cannot
+  // start the replica at all or must first move tens of gigabytes.
+  //
+  // Empty means the node was not asked to look, which is not the same as having
+  // nothing — an agent with no model directories configured behaves exactly as
+  // every agent did before this existed, and constrains nothing.
+  repeated LocalWeights weights = 11;
+
+  // storage is the disk behind those weights, one entry per filesystem.
+  //
+  // Reported so an operator can see a node filling up before a fetch dies at
+  // ninety percent, and so placement can refuse a node that cannot hold a
+  // model rather than discovering it during the download.
+  //
+  // Empty means the node was not asked to look, exactly as for weights, and
+  // constrains nothing.
+  repeated Storage storage = 12;
+
+  // host is what the machine is, so the control plane can have the mirror
+  // fetch software built for it rather than assuming every node matches the
+  // first.
+  Host host = 13;
+
+  // engines_probed says engine_versions is an answer. Without it an empty map
+  // meant "an agent too old to say" and constrained nothing — so a freshly
+  // built node with no engine installed yet was a candidate for every model.
+  bool engines_probed = 14;
+}
+
+// Host describes the operating system and processor of a node.
+message Host {
+  // From /etc/os-release: "ubuntu", "24.04", "noble".
+  string os_id = 1;
+  string os_version = 2;
+  string os_codename = 3;
+  // Debian's spelling, which is what package indexes are keyed by: "amd64",
+  // "arm64".
+  string arch = 4;
+  // The vector extensions inference cares about, of those the CPU has. A VM
+  // given QEMU's default CPU model has none of them, and runs, and is five
+  // times slower, and nothing says why.
+  repeated string cpu_flags = 5;
+  // nvidia_driver is the driver version, empty when there is none. A card
+  // without a driver is not a GPU anything can use.
+  string nvidia_driver = 6;
+}
+
+// Storage is one filesystem a node keeps weights on.
+//
+// Per filesystem rather than per configured directory: free space belongs to
+// the disk, and two model directories on one disk are one pool of bytes.
+message Storage {
+  // path is one directory on this filesystem, so the operator recognises it.
+  string path = 1;
+  // free_bytes is what a non-root process can write — not the root reserve,
+  // which is space the agent cannot have.
+  int64 free_bytes = 2;
+  int64 total_bytes = 3;
+  // shared marks a filesystem the node did not fill. Free space on one is
+  // shared with every other node using it, so two nodes each reporting 500 GB
+  // free on the same filer have 500 GB between them and not a terabyte.
+  bool shared = 4;
+}
+
+// RunningShard describes an engine already running on a node.
+//
+// It carries enough for the control plane to rebuild the replica it belongs to,
+// which is more than a shard id: the replica's identity, its pool and fabric,
+// and the leader endpoint clients reach it through.
+message RunningShard {
+  string replica_id = 1;
+  string shard_id = 2;
+  int32 rank = 3;
+  int32 nnodes = 4;
+  repeated int32 gpu_indices = 5;
+  repeated string hcas = 6;
+  int32 http_port = 7;
+  ShardStatus.State state = 8;
+
+  // model and profile name the catalog entry this shard was started from.
+  string model = 9;
+  string profile = 10;
+  // pool and fabric are echoed back from the StartShard that created it; the
+  // agent has no way to derive them.
+  string pool = 11;
+  string fabric = 12;
+
+  string leader_addr = 13;
+  int32 leader_port = 14;
+
+  // profile_fp is the fingerprint of the profile this shard was started from,
+  // echoed back from StartShard. The control plane keeps no record of a running
+  // replica across its own restart, so without this a shard adopted after a
+  // restart could not be compared with the catalog — which is exactly when the
+  // catalog is most likely to have been edited.
+  string profile_fp = 15;
+}
+
+message GPU {
+  int32 index = 1;
+  string uuid = 2;
+  string name = 3;
+  int32 memory_total_mb = 4;
+  int32 memory_used_mb = 5;
+  string compute_capability = 6;
+  int32 numa_node = 7;
+  string pci_bus_id = 8;
+  string driver_version = 9;
+  string cuda_version = 10;
+  // nvlink_peers is only populated by agents built with the nvml tag.
+  repeated int32 nvlink_peers = 11;
+  // problems keeps a GPU out of placement: ECC faults, or memory already held
+  // by another process.
+  repeated string problems = 12;
+  int32 temperature_c = 13;
+  int32 ecc_errors = 14;
+  // utilization_percent is how busy the card was over the last sample period.
+  // Zero when the driver is too old to report it, which is not the same as
+  // idle: the metric is simply absent rather than wrong.
+  int32 utilization_percent = 15;
+}
+
+message FabricAttachment {
+  // fabric is the operator-assigned name. It is not derived from the GID
+  // prefix: OpenSM defaults to fe80:: on every fabric, so a matching prefix
+  // proves nothing (plan §5.2).
+  string fabric = 1;
+  repeated string hcas = 2;
+  int32 rate_gbps = 3;
+}
+
+// Heartbeat says the node is alive and reports anything that changed cheaply.
+// LocalWeights is one model file or directory a node can see.
+message LocalWeights {
+  // path is what an engine would be given, so it can be compared against a
+  // model's source without interpretation.
+  string path = 1;
+  int64 size_bytes = 2;
+  // shared marks weights on a filesystem the node did not have to fill —
+  // reported so a fabric with a shared filer is not mistaken for one where
+  // every node happens to have fetched its own copy.
+  bool shared = 3;
+}
+
+message Heartbeat {
+  string node_id = 1;
+  // gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on
+  // a card after registration.
+  repeated int32 gpu_memory_used_mb = 2;
+  repeated string new_problems = 3;
+
+  // storage for the same reason the GPU memory is here: it changes while the
+  // agent runs, and a figure sampled once at registration is a figure that is
+  // wrong on exactly the node worth watching — the one busy filling up.
+  //
+  // Always sent, because it is a few numbers per filesystem.
+  repeated Storage storage = 4;
+
+  // weights only when they changed, because the list is long enough that
+  // sending it every few seconds would be paying for nothing most of the time.
+  //
+  // weights_changed is what says the list is meant: without it an empty list
+  // would be ambiguous between "this node now holds nothing" and "nothing to
+  // report", and the first would quietly empty a node's inventory.
+  bool weights_changed = 5;
+  repeated LocalWeights weights = 6;
+
+  // memory_total_mb for the same reason again: RAM is added to a virtual
+  // machine while it runs, or across a reboot the agent does not notice, and a
+  // figure read once at registration kept a node that had grown to 32 GB
+  // looking like the 7 GB machine it used to be. 0 means not read.
+  int64 memory_total_mb = 7;
+
+  // engine_versions only when they changed, like weights. An engine installed
+  // after the agent started is otherwise invisible to placement until
+  // somebody restarts the agent — which is a step nobody knows to take.
+  bool engines_changed = 8;
+  map<string, string> engine_versions = 9;
+
+  // provision is where installing the node's software stands, sent when it
+  // changed. The flag makes an empty list mean "nothing to install" rather
+  // than "nothing to say", as for weights.
+  bool provision_changed = 10;
+  repeated ProvisionStatus provision = 11;
+}
+
+// Provision is the software a node should have: which engines, at which
+// versions. Nothing more.
+//
+// Everything else is the node's to work out, because only the node knows it:
+// which build fits its hardware, where that build is published, what its
+// operating system is missing, and whether the result actually runs. The agent
+// converges on this — installs what is absent, checks what is there — and
+// reports each engine through ProvisionStatus. Nothing is taken on trust from
+// what happens to be on the machine: an engine is offered to placement only
+// once the agent has installed it and seen it run.
+message Provision {
+  repeated EngineWant engines = 1;
+}
+
+// EngineWant names one engine at one version.
+message EngineWant {
+  string engine = 1;  // "llamacpp"
+  string version = 2; // "b10991"
+}
+
+// ProvisionStatus is where one engine's installation stands on a node.
+message ProvisionStatus {
+  string engine = 1;
+  string version = 2;
+  string variant = 3;
+  // "installing", "ready", "failed", or "unsupported" when no build fits.
+  string state = 4;
+  // detail says what is happening or what went wrong, in words for a person.
+  string detail = 5;
+}
+
+// ShardStatus reports a shard's lifecycle transition.
+message ShardStatus {
+  string node_id = 1;
+  string replica_id = 2;
+  string shard_id = 3;
+  State state = 4;
+  string error = 5;
+  int32 exit_code = 6;
+  // last_output carries the tail of the engine log on a failure. When a model
+  // fails to load this is the only diagnosis available.
+  repeated string last_output = 7;
+
+  // fetch is how far a download has got, when the shard is waiting on one.
+  //
+  // The agent logs this already, on the node, where nobody is looking. From
+  // the control plane a replica pulling a hundred gigabytes and one that is
+  // wedged are the same thing — a shard that has been starting for ten
+  // minutes — and telling them apart is the difference between waiting and
+  // going to look.
+  FetchProgress fetch = 8;
+
+  enum State {
+    STATE_UNSPECIFIED = 0;
+    STATE_PENDING = 1;
+    STATE_STARTING = 2;
+    STATE_LOADING = 3;
+    STATE_READY = 4;
+    STATE_DRAINING = 5;
+    STATE_STOPPED = 6;
+    STATE_FAILED = 7;
+  }
+}
+
+// FetchProgress is a download in flight.
+message FetchProgress {
+  string source = 1;
+  int64 bytes_done = 2;
+  // bytes_total is 0 when the server did not say how large the file is, which
+  // is a real answer: a percentage would then be invented, and an invented
+  // percentage that stops moving is worse than a byte count that keeps rising.
+  int64 bytes_total = 3;
+}
+
+// StartShard launches one shard. It is idempotent by shard_id: the control
+// plane retries, and a repeat must not produce a second process.
+message StartShard {
+  string replica_id = 1;
+  string shard_id = 2;
+
+  int32 rank = 3;
+  int32 nnodes = 4;
+  string leader_addr = 5;
+  int32 leader_port = 6;
+
+  repeated int32 gpu_indices = 7;
+  // hcas are the adapters of the chosen fabric and nothing else. Handing a
+  // shard adapters from a second fabric lets NCCL pick one whose peer is
+  // unreachable, and the job hangs instead of failing (plan §5.3).
+  repeated string hcas = 8;
+  string mgmt_interface = 9;
+  int32 http_port = 10;
+
+  EngineProfile profile = 11;
+  int64 startup_timeout_ms = 12;
+
+  // pool and fabric are control plane concepts the agent cannot derive. They
+  // are sent so the agent can echo them back in RunningShard after a restart.
+  string pool = 13;
+  string fabric = 14;
+
+  // profile_fp identifies the version of the profile, so the agent can say
+  // which one a running shard was started from. The agent never interprets it.
+  string profile_fp = 15;
+}
+
+message EngineProfile {
+  string name = 1;
+  string engine = 2;
+  string gpu_class = 3;
+  int32 tp = 4;
+  int32 pp = 5;
+  int32 gpus_per_node = 6;
+  int32 nodes = 7;
+  repeated string args = 8;
+  map<string, string> env = 9;
+  string model_path = 10;
+  string served_model_name = 11;
+}
+
+// RemoveWeights deletes cached model files a node no longer needs.
+//
+// The paths are computed by the control plane from the catalog, but the agent
+// checks them again before unlinking anything: it deletes only files inside its
+// own cache directory whose names this system generated. A control plane that
+// is wrong, compromised or simply newer than the agent must not be able to
+// talk it into deleting a model an operator spent a week copying there.
+message RemoveWeights {
+  repeated string paths = 1;
+}
+
+// StopShard terminates a shard and frees its GPUs.
+message StopShard {
+  string shard_id = 1;
+}
+
+// DrainShard stops routing to a shard while letting in-flight work finish.
+message DrainShard {
+  string shard_id = 1;
+}
+
+// Ack confirms a registration or reports that a command could not be accepted.
+message Ack {
+  string message_id = 1;
+  bool ok = 2;
+  string error = 3;
+}
+
+// LogsRequest asks a node for the recent lines of one of its logs.
+//
+// Over the stream the agent already holds, because that is the one way into a
+// node every deployment has: the admin page otherwise means ssh through a jump
+// host to a machine whose address nobody remembers, to read a journal only
+// root may open.
+message LogsRequest {
+  // request_id pairs the reply with its request; several may be in flight.
+  string request_id = 1;
+  // source is "agent" for the agent's own log, or a shard id for that
+  // engine's output.
+  string source = 2;
+  int32 lines = 3;
+}
+
+// LogsReply answers a LogsRequest.
+message LogsReply {
+  string request_id = 1;
+  repeated string lines = 2;
+  // sources are every log this node can show, so a page can offer them
+  // without knowing what runs there.
+  repeated string sources = 3;
+  // error says why the lines are absent: an unknown source, say.
+  string error = 4;
+}

buf.gen.yaml

@@ -0,0 +1,12 @@
+version: v2
+managed:
+  enabled: true
+plugins:
+  - local: protoc-gen-go
+    out: api/gen
+    opt: paths=source_relative
+  - local: protoc-gen-connect-go
+    out: api/gen
+    opt: paths=source_relative
+inputs:
+  - directory: api/proto

buf.yaml

@@ -0,0 +1,16 @@
+version: v2
+modules:
+  - path: api/proto
+lint:
+  use:
+    - STANDARD
+  except:
+    # The standard names assume request/response pairs. This is one long-lived
+    # bidirectional stream carrying several kinds of message in each direction,
+    # so "AgentMessage" and "ControlMessage" say what they are and
+    # "ConnectRequest"/"ConnectResponse" would not.
+    - RPC_REQUEST_STANDARD_NAME
+    - RPC_RESPONSE_STANDARD_NAME
+breaking:
+  use:
+    - FILE

deploy/bootstrap/add-node.sh

@@ -0,0 +1,112 @@
+#!/bin/sh
+# add-node.sh — build a node and put it in the fleet, in one command.
+#
+# provision-node.sh and onboard-node.sh stay separate because the two halves
+# need different access: one the hypervisor, one the certificate authority.
+# That is a real boundary and this script does not erase it — it drives both
+# where one person holds both, which is the common case and was the case here.
+#
+# The reason to have it is not keystrokes. The VM name, the hostname, the node
+# id and the certificate's common name must all be the same string, and the
+# address appears in three places; split across two invocations run minutes
+# apart, that agreement is maintained by whoever is typing. Here it is stated
+# once.
+set -eu
+
+usage() {
+  cat >&2 <<'USAGE'
+usage: add-node.sh --name NODE --ip ADDR/CIDR --server URL [options]
+
+  --name NODE       node id, hostname and VM name; also the certificate's CN
+  --ip ADDR/CIDR    e.g. 10.0.0.145/24
+  --server URL      control plane, e.g. https://10.0.0.140:7443
+
+  --vmid N          Proxmox VMID (default: the last octet of --ip)
+  --cores N         (default 4)
+  --cpu TYPE        QEMU CPU model (default host; see provision-node.sh)
+  --memory MB       (default 8192)
+  --disk GB         (default 20)
+  --fabric NAME     InfiniBand fabric; repeatable as NAME:hca1,hca2.
+                    Omit for a node with no fabric.
+  --llamacpp-bin P  path to llama-server on the node
+  --vllm-bin P      path to vllm
+  --sglang-python P path to the python that has sglang
+  --binary PATH     llmgw binary to install (default: the one on PATH)
+  --pki DIR         certificate authority (default /etc/llmgw/pki)
+  --ssh-user U      (default ilker)
+  --provision-only  build the VM and stop
+  --onboard-only    skip provisioning; the machine already exists
+USAGE
+  exit 2
+}
+
+NAME= IP= SERVER= VMID= CORES=4 MEMORY=8192 DISK=20 CPU=
+BINARY= PKI=/etc/llmgw/pki SSH_USER=ilker
+PROVISION=1 ONBOARD=1
+set -- "$@"
+PASS_THROUGH=
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --name) NAME="$2"; shift 2;;
+    --ip) IP="$2"; shift 2;;
+    --server) SERVER="$2"; shift 2;;
+    --vmid) VMID="$2"; shift 2;;
+    --cores) CORES="$2"; shift 2;;
+    --cpu) CPU="$2"; shift 2;;
+    --memory) MEMORY="$2"; shift 2;;
+    --disk) DISK="$2"; shift 2;;
+    --binary) BINARY="$2"; shift 2;;
+    --pki) PKI="$2"; shift 2;;
+    --ssh-user) SSH_USER="$2"; shift 2;;
+    --provision-only) ONBOARD=0; shift;;
+    --onboard-only) PROVISION=0; shift;;
+    # Everything the onboarding step understands and this one does not need to.
+    --fabric|--llamacpp-bin|--vllm-bin|--sglang-python|--ib-root|--nvidia-smi|--mgmt-addr|--mgmt-if)
+      PASS_THROUGH="$PASS_THROUGH $1 $2"; shift 2;;
+    -h|--help) usage;;
+    *) echo "unknown option: $1" >&2; usage;;
+  esac
+done
+[ -n "$NAME" ] && [ -n "$IP" ] && [ -n "$SERVER" ] || usage
+
+# The address without its prefix, which is what onboarding and ssh want.
+ADDR="${IP%%/*}"
+# The last octet as the VMID by default: it keeps `qm list` and `ctl nodes` in
+# the same order, and one fewer number to choose is one fewer to get wrong.
+[ -n "$VMID" ] || VMID="${ADDR##*.}"
+
+here() { dirname "$0"; }
+PROVISION_SH="$(here)/provision-node.sh"
+ONBOARD_SH="$(here)/onboard-node.sh"
+[ -x "$PROVISION_SH" ] || PROVISION_SH="$(command -v llmgw-provision-node || true)"
+[ -x "$ONBOARD_SH" ] || ONBOARD_SH="$(command -v llmgw-onboard-node || true)"
+
+if [ "$PROVISION" = 1 ]; then
+  [ -x "$PROVISION_SH" ] || { echo "cannot find provision-node.sh" >&2; exit 1; }
+  echo "== 1/2  building $NAME as VM $VMID"
+  # shellcheck disable=SC2086
+  "$PROVISION_SH" --vmid "$VMID" --name "$NAME" --ip "$IP" \
+    --cores "$CORES" --memory "$MEMORY" --disk "$DISK" ${CPU:+--cpu "$CPU"}
+fi
+
+if [ "$ONBOARD" = 0 ]; then
+  echo "$NAME built. Onboard it from the CA host when you are ready."
+  exit 0
+fi
+
+[ -x "$ONBOARD_SH" ] || { echo "cannot find onboard-node.sh" >&2; exit 1; }
+echo
+echo "== 2/2  onboarding $NAME"
+# Onboarding waits for ssh itself, which is what makes running the two halves
+# back to back safe: cloud-init is still regenerating host keys and restarting
+# sshd for the first minute of a new machine's life.
+# shellcheck disable=SC2086
+"$ONBOARD_SH" --node "$NAME" --addr "$ADDR" --server "$SERVER" \
+  --pki "$PKI" --ssh-user "$SSH_USER" \
+  ${BINARY:+--binary "$BINARY"} $PASS_THROUGH
+
+echo
+echo "Check it with: llmgw ctl nodes"
+echo
+echo "The agent installs its engines itself; the node takes no model until they"
+echo "are installed and seen to run. 'llmgw ctl nodes' shows where that stands."

deploy/bootstrap/onboard-node.sh

@@ -0,0 +1,289 @@
+#!/usr/bin/env bash
+#
+# Give a machine an identity in an llmgw cluster.
+#
+# Run on the host that holds the certificate authority, because that is the one
+# thing that cannot be delegated: this script issues the credential by which the
+# control plane will believe the node is who it says it is.
+#
+# What it does, in the order it matters:
+#   1. issue a client certificate whose common name is the node id
+#   2. copy the CA root, the certificate and its key to the machine
+#   3. trust the root system-wide, so other tooling there stops complaining
+#   4. install the binary, the configuration and the unit
+#   5. start the agent
+#
+# The node id, the VM name, the hostname and the certificate common name are all
+# the same string. They do not have to be, and every time they are not, someone
+# spends an afternoon working out which of the four a message is talking about.
+set -euo pipefail
+
+usage() {
+  cat >&2 <<USAGE
+usage: $0 --node ID --addr HOST --server URL [options]
+
+  --node ID         node id, hostname and certificate common name
+  --addr HOST       how to reach the machine over ssh
+  --server URL      the control plane's agent endpoint, e.g. https://10.0.0.140:7443;
+                    several, comma separated, for a node that fails over
+  --fabric NAME     InfiniBand fabric this node is attached to; repeatable as
+                    --fabric NAME:hca1,hca2 . Omit for a node with no fabric,
+                    which can host single-node replicas and will never be given
+                    a multi-node one.
+  --mgmt-addr ADDR  address engines are reachable on (default: the node's
+                    address on the route to --server, found by the agent)
+  --mgmt-if NAME    interface NCCL bootstraps over (default: the one holding
+                    that address)
+  --llamacpp-bin P  use this llama-server instead of the one the agent installs
+  --vllm-bin P      path to the vllm entry point (not yet installed by the agent)
+  --sglang-python P python interpreter of the SGLang virtualenv
+  --nvidia-smi PATH where nvidia-smi lives, when it is not on PATH. Containerized
+                    driver installs routinely put it somewhere else.
+  --ib-root DIR     the sysfs InfiniBand directory (default /sys/class/infiniband)
+  --pki DIR         where the authority lives (default /etc/llmgw/pki)
+  --binary PATH     llmgw binary to install (default: the one running this).
+                    Releases carry static linux/amd64 and linux/arm64 builds;
+                    downloading one is better than copying whatever happens to
+                    be on the machine you are running this from.
+  --releases URL    where release binaries are downloaded from (default the
+                    project's Gitea releases)
+  --ssh-user NAME   (default ilker)
+USAGE
+  exit 2
+}
+
+NODE= ADDR= SERVER= MGMT_OVERRIDE= MGMT_IF_OVERRIDE=
+NVIDIA_SMI= IB_ROOT=/sys/class/infiniband
+LLAMACPP_BIN= VLLM_BIN= SGLANG_PYTHON=
+SSH_WAIT=180
+PKI=/etc/llmgw/pki BINARY= SSH_USER=ilker
+RELEASES=https://gitea.mynodes.xyz/ilker/llmgw/releases/download
+FABRIC_NAME= FABRIC_HCAS="mlx5_0,mlx5_1"
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --node) NODE="$2"; shift 2;;
+    --addr) ADDR="$2"; shift 2;;
+    --server) SERVER="$2"; shift 2;;
+    --fabric) FABRIC_NAME="${2%%:*}"; [ "${2#*:}" != "$2" ] && FABRIC_HCAS="${2#*:}"; shift 2;;
+    --mgmt-addr) MGMT_OVERRIDE="$2"; shift 2;;
+    --mgmt-if) MGMT_IF_OVERRIDE="$2"; shift 2;;
+    --llamacpp-bin) LLAMACPP_BIN="$2"; shift 2;;
+    --vllm-bin) VLLM_BIN="$2"; shift 2;;
+    --sglang-python) SGLANG_PYTHON="$2"; shift 2;;
+    --nvidia-smi) NVIDIA_SMI="$2"; shift 2;;
+    --ib-root) IB_ROOT="$2"; shift 2;;
+    --pki) PKI="$2"; shift 2;;
+    --binary) BINARY="$2"; shift 2;;
+    --releases) RELEASES="$2"; shift 2;;
+    --ssh-user) SSH_USER="$2"; shift 2;;
+    *) usage;;
+  esac
+done
+[ -n "$NODE" ] && [ -n "$ADDR" ] && [ -n "$SERVER" ] || usage
+BINARY="${BINARY:-$(command -v llmgw)}"
+[ -x "$BINARY" ] || { echo "no llmgw binary; pass --binary" >&2; exit 1; }
+[ -f "$PKI/ca-key.pem" ] || { echo "no authority in $PKI; run 'llmgw ctl cert init-ca' first" >&2; exit 1; }
+
+# accept-new, not "no": a machine provisioned a minute ago has a host key
+# nobody has seen, and refusing it stopped onboarding dead with "Host key
+# verification failed" — an error about ssh, in the middle of a script whose
+# whole job is to talk to a new machine. accept-new learns an unknown key and
+# still refuses a changed one, which is the distinction that matters: the risk
+# being guarded against is a substituted machine, not a new one.
+SSHOPTS="-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new"
+SSH="ssh $SSHOPTS $SSH_USER@$ADDR"
+say() { printf '  %s\n' "$*"; }
+
+echo "onboarding $NODE at $ADDR"
+
+# Waiting rather than failing: cloud-init restarts sshd after it regenerates
+# the host keys, so a machine that answered a moment ago refuses connections
+# for a few seconds. Observed while onboarding node05 — the agent was already
+# registered and healthy while ssh was refusing.
+wait_for_ssh() {
+  deadline=$(( $(date +%s) + SSH_WAIT ))
+  until $SSH true 2>/dev/null; do
+    [ "$(date +%s)" -lt "$deadline" ] || {
+      echo "cannot reach $SSH_USER@$ADDR over ssh after ${SSH_WAIT}s" >&2
+      exit 1
+    }
+    sleep 5
+  done
+}
+wait_for_ssh
+
+# 1. The certificate. Issued fresh every time: re-onboarding a machine should
+#    not depend on finding an old key, and the old one keeps working until it
+#    is revoked, which is a separate decision.
+say "issuing a certificate"
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT
+"$BINARY" ctl cert issue-client "$NODE" --dir "$PKI" --out "$TMP" >/dev/null
+
+# 2 and 3. Identity, and trusting the root so future tooling on that machine
+#    does not treat our certificates as strangers.
+say "installing the certificate and trusting the root"
+scp -q $SSHOPTS "$PKI/ca.pem" "$TMP/$NODE.pem" "$TMP/$NODE-key.pem" "$SSH_USER@$ADDR:/tmp/"
+$SSH "set -e
+  sudo install -d -m 0755 /etc/llmgw
+  sudo install -m 0644 /tmp/ca.pem /etc/llmgw/ca.pem
+  sudo install -m 0644 /tmp/$NODE.pem /etc/llmgw/agent.pem
+  sudo install -m 0600 /tmp/$NODE-key.pem /etc/llmgw/agent-key.pem
+  sudo chown root:root /etc/llmgw/*
+  sudo install -m 0644 /tmp/ca.pem /usr/local/share/ca-certificates/llmgw-ca.crt
+  sudo update-ca-certificates >/dev/null 2>&1 || true
+  rm -f /tmp/ca.pem /tmp/$NODE.pem /tmp/$NODE-key.pem"
+
+# 4. The binary, the configuration and the unit.
+# The node fetches the release CI built for this version, for its own
+# architecture, and checks it against the release's checksums — rather than
+# being handed whatever binary is on the machine running this script. A build
+# that is not a release (a developer's, with commits past the tag) has no such
+# file, and is copied as before.
+TAG="$("$BINARY" --version 2>/dev/null | awk '{print $NF}')"
+if printf '%s' "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
+  say "installing llmgw $TAG from the release"
+  $SSH "set -e
+    arch=\$(dpkg --print-architecture)
+    name=llmgw-$TAG-linux-\$arch
+    cd /tmp && rm -f \$name checksums.txt
+    curl -fsSL -o \$name '$RELEASES/$TAG/'\$name
+    curl -fsSL -o checksums.txt '$RELEASES/$TAG/checksums.txt'
+    grep \" \$name\\\$\" checksums.txt | sha256sum -c - >/dev/null
+    sudo install -m 0755 \$name /usr/local/bin/llmgw
+    rm -f \$name checksums.txt"
+else
+  say "installing the binary (${TAG:-unversioned}, not a release)"
+  scp -q $SSHOPTS "$BINARY" "$SSH_USER@$ADDR:/tmp/llmgw"
+  $SSH 'sudo install -m 0755 /tmp/llmgw /usr/local/bin/llmgw && rm -f /tmp/llmgw'
+fi
+
+say "writing the configuration"
+NODE_IS_HOSTNAME=no
+[ "$($SSH hostname 2>/dev/null)" = "$NODE" ] && NODE_IS_HOSTNAME=yes
+# Only what the node cannot find out for itself. Its name is its hostname, its
+# address is the one on the route to the control plane, llama-server is looked
+# for on PATH and where release archives are unpacked, GPUs are there when
+# nvidia-smi is, and memory, disk and engines are re-read while it runs. Every
+# setting written here is one that has to be kept true by hand — so an option
+# below is written only when it was given, as an override.
+{
+  echo "# $NODE, written by onboard-node.sh."
+  # Several addresses, comma separated, are every control plane the node may
+  # fail over to; a standby refuses it and it moves on to the next.
+  if [ "${SERVER#*,}" != "$SERVER" ]; then
+    echo "servers:"
+    printf '%s\n' "$SERVER" | tr ',' '\n' | sed 's/^ *//; s/^/  - "/; s/$/"/'
+  else
+    echo "server: \"$SERVER\""
+  fi
+  echo
+  echo "tls:"
+  echo "  ca: /etc/llmgw/ca.pem"
+  echo "  cert: /etc/llmgw/agent.pem"
+  echo "  key: /etc/llmgw/agent-key.pem"
+  # The certificate's common name is --node, and the control plane refuses a
+  # stream whose node id differs from it. Said explicitly only when the
+  # hostname would say something else.
+  [ "$NODE_IS_HOSTNAME" = yes ] || echo "node_id: $NODE"
+  [ -n "$MGMT_OVERRIDE" ] && echo "mgmt_addr: \"$MGMT_OVERRIDE\""
+  [ -n "$MGMT_IF_OVERRIDE" ] && echo "mgmt_interface: $MGMT_IF_OVERRIDE"
+  if [ -n "$FABRIC_NAME" ]; then
+    # Fabric membership is the one thing that cannot be found: OpenSM's default
+    # subnet prefix is the same on every fabric, so a matching prefix proves
+    # nothing and a wrong guess hangs the first all-reduce.
+    echo "fabrics:"
+    echo "  - name: $FABRIC_NAME"
+    echo "    hcas: [$(echo "$FABRIC_HCAS" | sed 's/,/, /g')]"
+  fi
+  if [ -n "$NVIDIA_SMI" ] || [ "$IB_ROOT" != /sys/class/infiniband ]; then
+    echo "discovery:"
+    [ -n "$NVIDIA_SMI" ] && echo "  nvidia_smi: \"$NVIDIA_SMI\""
+    [ "$IB_ROOT" != /sys/class/infiniband ] && echo "  ib_root: $IB_ROOT"
+  fi
+  if [ -n "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then
+    echo "engines:"
+    [ -n "$LLAMACPP_BIN" ] && printf '  llamacpp:\n    bin: "%s"\n' "$LLAMACPP_BIN"
+    [ -n "$VLLM_BIN" ] && printf '  vllm:\n    bin: "%s"\n' "$VLLM_BIN"
+    [ -n "$SGLANG_PYTHON" ] && printf '  sglang:\n    python: "%s"\n' "$SGLANG_PYTHON"
+  fi
+  true
+} > "$TMP/agent.yaml"
+scp -q "$TMP/agent.yaml" "$SSH_USER@$ADDR:/tmp/agent.yaml"
+
+cat > "$TMP/llmgw-agent.service" <<'UNIT'
+[Unit]
+Description=llmgw node agent
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+ExecStart=/usr/local/bin/llmgw agent --config /etc/llmgw/agent.yaml
+# Root, because the agent launches engines and needs device access. Engines
+# outlive a restart of this unit and are adopted again on the next start, so
+# KillMode must not take the whole process group with it.
+KillMode=process
+Restart=on-failure
+RestartSec=2s
+StateDirectory=llmgw-agent
+
+[Install]
+WantedBy=multi-user.target
+UNIT
+scp -q "$TMP/llmgw-agent.service" "$SSH_USER@$ADDR:/tmp/"
+
+# An engine path that is wrong, or right but unrunnable, is the failure this
+# node type is most likely to have, and it is invisible: the agent registers
+# healthy and every shard placed on it dies with "executable file not found"
+# or a missing shared library. Checked here because it is cheap here and
+# expensive later — llama-server on node05 was present, executable, and could
+# not start for want of libgomp1, and the node reported ready throughout.
+check_engine() {
+  [ -n "$2" ] || return 0
+  if $SSH "test -x '$2'" 2>/dev/null; then
+    if $SSH "'$2' --version >/dev/null 2>&1 || '$2' --help >/dev/null 2>&1"; then
+      say "$1: $2 runs"
+      return 0
+    fi
+    ENGINE_PROBLEMS="$ENGINE_PROBLEMS
+  $1 at $2 exists but will not run. Usually a missing shared library:
+    ssh $SSH_USER@$ADDR ldd $2 | grep 'not found'"
+    return 0
+  fi
+  ENGINE_PROBLEMS="$ENGINE_PROBLEMS
+  $1 is configured at $2, which is not an executable on $NODE."
+}
+ENGINE_PROBLEMS=
+if [ -n "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then
+  say "checking the engines"
+  check_engine llamacpp "$LLAMACPP_BIN"
+  check_engine vllm "$VLLM_BIN"
+  check_engine sglang "$SGLANG_PYTHON"
+fi
+
+say "starting the agent"
+$SSH 'set -e
+  sudo install -m 0644 -o root -g root /tmp/agent.yaml /etc/llmgw/agent.yaml
+  sudo install -m 0644 /tmp/llmgw-agent.service /etc/systemd/system/llmgw-agent.service
+  rm -f /tmp/agent.yaml /tmp/llmgw-agent.service
+  sudo systemctl daemon-reload
+  sudo systemctl enable --now llmgw-agent >/dev/null
+  sudo systemctl restart llmgw-agent'
+
+echo
+if [ -z "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then
+  say "the agent now installs the engines the control plane asks for: the"
+  say "build for this machine, checked against its publisher's checksum and run"
+  say "before the node is offered any model. 'llmgw ctl nodes' shows progress."
+fi
+if [ -n "$ENGINE_PROBLEMS" ]; then
+  echo "$NODE is onboarded but an engine will not run:$ENGINE_PROBLEMS"
+  echo
+  echo "The node will register as ready regardless. Fix this before a replica"
+  echo "is placed on it, or the shard fails and the reason is three logs away."
+  echo
+fi
+echo "$NODE onboarded. It should appear in 'llmgw ctl nodes' within a few seconds."
+echo "If it does not, the control plane's log says why — a certificate whose"
+echo "common name does not match the claimed node id is refused by design."

deploy/bootstrap/provision-node.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+#
+# Create a VM on Proxmox for a new llmgw node.
+#
+# Run on the Proxmox host. It does the half of onboarding that is about
+# hardware; onboard-node.sh does the half that is about identity, and the two
+# are separate because they are done by different people with different access.
+#
+# Idempotent only in the sense that it refuses to touch an existing VMID. A node
+# that already exists is not something a provisioning script should have an
+# opinion about.
+set -euo pipefail
+
+usage() {
+  cat >&2 <<USAGE
+usage: $0 --vmid N --name NAME --ip ADDR [options]
+
+  --vmid N          Proxmox VM id; must be free
+  --name NAME       VM name; use the node id, so the machine, the hostname and
+                    the certificate common name are the same string
+  --ip ADDR         address with prefix, e.g. 10.0.0.144/24
+  --gw ADDR         gateway (default 10.0.0.1)
+  --cores N         (default 4)
+  --cpu TYPE        QEMU CPU model (default host)
+  --memory MB       (default 8192)
+  --disk GB         (default 20)
+  --bridge NAME     (default vmbr0)
+  --storage NAME    (default local-lvm)
+  --image PATH      cloud image (default Ubuntu 24.04 in the ISO store)
+  --user NAME       cloud-init user (default ilker)
+  --sshkeys PATH    authorized_keys to install (default /root/.ssh/authorized_keys)
+USAGE
+  exit 2
+}
+
+VMID= NAME= IP= GW=10.0.0.1 CORES=4 MEMORY=8192 DISK=20
+# host, not Proxmox's kvm64 default. kvm64 hides AVX, AVX2, FMA and even
+# SSE4.2 from the guest, and an inference node without vector instructions is
+# not slightly slower, it is unusable: llama.cpp falls back to its scalar
+# backend and a 1.5B model dropped from 39 tokens a second to 5.8, with prompt
+# processing 19x worse. It cost an afternoon to find because nothing is broken
+# — every number is simply small.
+#
+# The trade is live migration: a VM pinned to the host CPU cannot move to a
+# machine with a different one. For a node whose job is arithmetic, that is
+# the right way round.
+CPU=host
+BRIDGE=vmbr0 STORAGE=local-lvm USER=ilker
+IMAGE=/var/lib/vz/template/iso/noble-server-cloudimg-amd64.img
+SSHKEYS=/root/.ssh/authorized_keys
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --vmid) VMID="$2"; shift 2;;
+    --name) NAME="$2"; shift 2;;
+    --ip) IP="$2"; shift 2;;
+    --gw) GW="$2"; shift 2;;
+    --cores) CORES="$2"; shift 2;;
+    --cpu) CPU="$2"; shift 2;;
+    --memory) MEMORY="$2"; shift 2;;
+    --disk) DISK="$2"; shift 2;;
+    --bridge) BRIDGE="$2"; shift 2;;
+    --storage) STORAGE="$2"; shift 2;;
+    --image) IMAGE="$2"; shift 2;;
+    --user) USER="$2"; shift 2;;
+    --sshkeys) SSHKEYS="$2"; shift 2;;
+    *) usage;;
+  esac
+done
+[ -n "$VMID" ] && [ -n "$NAME" ] && [ -n "$IP" ] || usage
+
+if qm status "$VMID" >/dev/null 2>&1; then
+  echo "VM $VMID already exists. Pick a free id; this script will not adopt one." >&2
+  exit 1
+fi
+[ -f "$IMAGE" ] || { echo "no cloud image at $IMAGE" >&2; exit 1; }
+
+echo "creating $NAME as VM $VMID at $IP"
+qm create "$VMID" \
+  --name "$NAME" --cores "$CORES" --memory "$MEMORY" --cpu "$CPU" \
+  --net0 "virtio,bridge=$BRIDGE" --ostype l26 --agent enabled=1
+
+# Imported rather than installed: a cloud image plus cloud-init is the whole
+# reason a node takes a minute to build instead of an afternoon.
+qm importdisk "$VMID" "$IMAGE" "$STORAGE" >/dev/null
+qm set "$VMID" --scsihw virtio-scsi-pci --scsi0 "$STORAGE:vm-$VMID-disk-0,discard=on,ssd=1" >/dev/null
+qm disk resize "$VMID" scsi0 "${DISK}G" >/dev/null
+qm set "$VMID" --ide2 "$STORAGE:cloudinit" --boot order=scsi0 >/dev/null
+qm set "$VMID" \
+  --ipconfig0 "ip=$IP,gw=$GW" \
+  --nameserver "$GW 8.8.8.8" \
+  --ciuser "$USER" \
+  --sshkeys "$SSHKEYS" >/dev/null
+
+qm start "$VMID"
+echo "started. cloud-init takes a minute; then run onboard-node.sh from the CA host."

deploy/examples/agent.yaml

@@ -0,0 +1,114 @@
+# llmgw agent configuration, one per GPU node.
+#
+# The agent dials out to the control plane and keeps one stream open. Engines
+# keep serving if the agent restarts: the router talks to them directly, and
+# orphan adoption picks them up again on the next start.
+
+# Everything a node can find out about itself, it does: its id is its hostname,
+# its address is the one on the route to the control plane, llama-server is
+# looked for on PATH and in /opt/llama.cpp, GPUs are there when nvidia-smi is,
+# weights are fetched into /var/lib/llmgw-weights and looked for there, and
+# memory, disk and installed engines are re-read while the agent runs. So a
+# plain CPU node needs only the two blocks below, and the same file on every
+# node. What follows them are overrides, for when the defaults are wrong.
+
+server: "http://llmgw-server.internal:7443"
+
+# The agent's half of mutual TLS. The certificate's common name must be this
+# node's id — its hostname, unless node_id says otherwise — because the control
+# plane compares the two and refuses the stream when they disagree:
+#
+#   llmgw ctl cert issue-client gpu-a-01 --dir /etc/llmgw/pki
+#
+# With this set, `server` above must be an https:// address — an agent holding
+# certificates and dialling http:// is refused at startup rather than connecting
+# in the clear without noticing.
+#
+# tls:
+#   ca: /etc/llmgw/ca.pem
+#   cert: /etc/llmgw/agent.pem
+#   key: /etc/llmgw/agent-key.pem
+
+# ---- overrides ---------------------------------------------------------------
+
+# node_id: gpu-a-01          # when it is not the hostname
+
+# The address engines are reachable on, and the address NCCL bootstraps over.
+# Never a fabric address: NCCL rendezvouses over TCP before moving to RDMA.
+# mgmt_addr: "10.0.0.11"
+# mgmt_interface: eth0
+
+# Free-form, for a profile's node_selector. Nothing needs them; do not restate
+# what is measured (memory, GPUs, engines), because a label goes stale.
+# labels:
+#   rack: r12
+
+heartbeat_interval: 5s
+reconnect_base: 1s
+reconnect_max: 30s
+
+# Fabric labelling is explicit and there is no way around it. OpenSM's default
+# subnet prefix is fe80:: on every fabric, so a matching prefix proves nothing
+# and a wrong guess produces a replica that starts, passes health checks and
+# then hangs on its first all-reduce (plan §5.2).
+#
+# A node with no fabrics is its own single-node island: it can host single-node
+# replicas and will never be given a multi-node one.
+fabrics:
+  - name: ib-a
+    hcas: [mlx5_0, mlx5_1, mlx5_2, mlx5_3]
+    rate_gbps: 200
+
+discovery:
+  # Worth setting where the driver lives outside the default paths, which
+  # containerized driver installs often do.
+  nvidia_smi: ""
+  ib_root: /sys/class/infiniband
+  # NVML is the only source of NVLink topology, which is what keeps a
+  # tensor-parallel group off the PCIe path inside a node. It needs a binary
+  # built with `-tags nvml`; otherwise this silently falls back to nvidia-smi.
+  prefer_nvml: true
+
+# Where this node keeps model weights besides /var/lib/llmgw-weights, which is
+# where fetched weights land and is always looked in. Storage differs at
+# every installation and llmgw deliberately does not own it: a Lustre mount, an
+# image exported over the fabric with SRP or NVMe-oF, local NVMe, or a copy
+# made by hand all look identical from here.
+#
+# What it buys is placement preferring a node that already holds the model over
+# one that must first move tens of gigabytes. That matters most where fabrics
+# are isolated from one another, because weights in one fabric are not weights
+# in another and the only path between them is the management network.
+#
+# model_dirs:
+#   - /var/lib/llmgw-models
+#   - /mnt/lustre/models
+
+# The agent installs the engines the control plane names (see `engines:` in
+# server.yaml) under /opt/llmgw/engines: the build for this machine — CUDA when
+# an NVIDIA driver is loaded — checked against its publisher's checksum, with
+# the OS packages it needs, and run before the node takes any model. llama.cpp
+# is installed this way today. Naming a binary below opts that engine out and
+# uses yours instead; vLLM and SGLang still have to be named until the agent
+# installs them too.
+# engines_dir: /opt/llmgw/engines
+
+engines:
+  vllm:
+    bin: /opt/venvs/vllm/bin/vllm
+    # Which multi-node mechanism works depends on the vLLM version, and the
+    # flags have changed more than once. Check the version you deploy.
+    launch_mode: ray      # "ray" or "mp"
+    ray_bin: /opt/venvs/vllm/bin/ray
+  sglang:
+    # SGLang is launched as a module, which is its documented entry point.
+    python: /opt/venvs/sglang/bin/python
+
+supervisor:
+  work_dir: /var/lib/llmgw-agent
+  # Engine output is the only diagnosis when a model fails to load, so keep a
+  # useful tail of it.
+  log_lines: 512
+  # Engines need time to free GPU memory cleanly before being killed.
+  stop_timeout: 60s
+  health_interval: 2s

deploy/examples/server.yaml

@@ -7,7 +7,91 @@
 # __ for nesting:  LLMGW_ROUTER__RETRIES=3
 
 listen: ":8080"        # client-facing OpenAI API
-admin_listen: ":9090"  # metrics, health, pprof — do not expose this publicly
+admin_listen: "127.0.0.1:9090"  # dashboard, management API, pprof — keep it local
+# Metrics only, on their own port, so Prometheus can reach them without pprof
+# being reachable too. Empty means off; metrics stay on admin_listen either way.
+# metrics_listen: "0.0.0.0:9091"
+# Where node agents connect. Separate from the client port so the fleet-facing
+# surface can be firewalled independently.
+agent_listen: ":7443"
+
+# Mutual TLS on the agent port (plan §5.6). With it, a node's identity is taken
+# from its certificate's common name and an agent claiming a different node id
+# is refused. Without it the port is plaintext HTTP/2 and anyone who can reach
+# it can claim to be any node, so it must stay on a trusted network.
+#
+# These certificates come from llmgw's own authority, not a public one: public
+# authorities stopped issuing the clientAuth usage, so no Let's Encrypt
+# certificate can authenticate an agent.
+#
+#   llmgw ctl cert init-ca --dir /etc/llmgw/pki
+#   llmgw ctl cert issue-server llmgw-ctl --host 10.0.0.140 --dir /etc/llmgw/pki
+#
+# Every address an agent might dial has to appear in --host, bare IP addresses
+# included: hostname verification reads subject alternative names and ignores
+# the common name entirely.
+#
+# This covers the fleet-facing port only. The client-facing API above is public
+# and belongs behind a publicly trusted certificate.
+#
+# agent_tls:
+#   ca: /etc/llmgw/ca.pem
+#   cert: /etc/llmgw/server.pem
+#   key: /etc/llmgw/server-key.pem
+
+# The public page at "/". Someone who reaches this address without a key needs a
+# route to getting one, and the gateway cannot guess who to send them to.
+#
+# landing:
+#   contact: "platform@example.com"   # or an https:// URL, or "#platform on Slack"
+#   notice: "Internal use only."
+
+# API keys on the client-facing port (plan §7.9). Off by default, because a
+# gateway on a trusted network with no keys issued still has to serve — but
+# nothing else protects that port, so turning it on is what makes it safe to
+# expose.
+#
+#   llmgw keys init --store /var/lib/llmgw/llmgw.db
+#   llmgw keys create --store /var/lib/llmgw/llmgw.db --project default "laptop"
+#
+# The key is printed once. Only its hash is stored, so there is nothing to
+# recover it from.
+#
+# auth:
+#   enabled: true
+#   store: /var/lib/llmgw/llmgw.db
+#   # How long a resolved key is trusted before being read again. This is the
+#   # price of keeping the database off the request path: it bounds how long a
+#   # revoked key keeps working.
+#   cache_ttl: 30s
+#   # How long a rejected key is remembered as rejected, so a client looping
+#   # with a bad key cannot become a database read per request.
+#   negative_ttl: 5s
+#
+#   # Serve the signup form. What it creates is a pending account that can do
+#   # nothing until `llmgw users approve` places it in a project, so this is not
+#   # a way in — it only moves key creation from you to the person who needs one.
+#   # Off, there is no form and you issue keys with `llmgw keys create --claim`.
+#   signup_open: true
+#
+#   # What each model costs, in micro-units per thousand tokens. A unit is
+#   # whatever you decide: self-hosted models have no per-token price, so it can
+#   # be GPU-seconds, a notional rate, or 1000 everywhere — which turns budgets
+#   # into plain token quotas.
+#   #
+#   # A model priced at zero is exempt from budget checks, which is what lets a
+#   # key with nothing left still be served a cheap fallback.
+#   default_price: { prompt_per_1k: 0, completion_per_1k: 0 }
+#   pricing:
+#     llama-3.3-70b: { prompt_per_1k: 1000, completion_per_1k: 3000 }
+#     qwen2.5-32b:   { prompt_per_1k: 400,  completion_per_1k: 1200 }
+#
+#   # Usage rows are written in batches off the request path. Beyond buffer_size
+#   # they are dropped rather than blocking a request: losing some accounting is
+#   # bad, turning a stalled database into an outage is worse.
+#   flush_interval: 200ms
+#   batch_size: 500
+#   buffer_size: 8192
 
 router:
   scrape_interval: 2s
@@ -29,6 +113,39 @@ router:
   failure_threshold: 3
 
   default_profile: chat        # chat | batch | long-context | debug
+
+  # Approximate prefix routing (plan §7.5 tier 1): send a request to the replica
+  # that has probably already prefilled its prefix.
+  #
+  # It earns its keep when the working set does not fit in one replica's cache.
+  # With a handful of prompts and a large cache, scattering is free — every
+  # replica ends up holding everything — and this only adds constraint. Measured
+  # with sim-engines whose caches were too small for the working set: cache hits
+  # went from 22.5% to 81.7%.
+  #
+  # Watch llmgw_prefix_decisions_total to tune it. A high
+  # load_balance_override count means the guard is doing its job and affinity is
+  # not getting a chance; a high cache_miss_min_load means match_ratio or
+  # min_match is too strict for your traffic.
+  prefix:
+    enabled: false
+    # The shortest match worth acting on, in bytes. Below this the overlap is
+    # coincidence — two English prompts share "The".
+    min_match: 64
+    # The share of the prompt that must match. A short shared header on a very
+    # long prompt saves almost no prefill.
+    match_ratio: 0.3
+    # The imbalance guard, and both must trip before affinity is abandoned.
+    # Absolute alone fires on an idle cluster where one replica has two requests
+    # and another none; relative alone fires on a busy one where 200 against 180
+    # is a rounding error. Without the guard the cache hit rate rises while p99
+    # collapses.
+    balance_abs: 8
+    balance_rel: 1.5
+    # An engine evicts on its own schedule, so an old record may describe a
+    # cache that is gone.
+    ttl: 10m
+    max_nodes: 100000
   session_header: x-session-id # affinity key; falls back to the `user` field
   # Ask engines for usage even when the client did not, so token accounting
   # works regardless of client behaviour. The extra chunk is dropped on the way
@@ -48,9 +165,9 @@ router:
   # traffic, because an engine queue is FIFO and knows nothing about tenants.
   admission:
     enabled: true
-    # Until authentication lands, the band comes from a header and is trusted.
-    # Do not expose this port to callers who should not be able to promote
-    # themselves.
+    # With auth on, the key decides the band and this header is ignored. With
+    # auth off the header is trusted, so any client can promote itself into the
+    # top band — do not expose the port in that configuration.
     band_header: x-llmgw-band
     default_band: gold
     # A band waits once pool saturation passes its ceiling, so the cluster
@@ -79,27 +196,79 @@ router:
     kv_ceiling: 0.9
     session_load_factor: 1.25
 
+# How aggressively the control plane reconciles.
+# The engines every node installs, and at which version. Unset keeps the
+# versions this release was checked against (llama.cpp b10991). Each agent
+# picks the build for its own hardware and installs it itself; changing a
+# version here reaches every node on its next connect.
+# engines:
+#   llamacpp: b10991
+
+control:
+  interval: 5s
+  base_backoff: 10s
+  max_backoff: 10m
+  # Node expiry is staged deliberately: a network blip must not tear down
+  # replicas that are still serving, and rebuilding a multi-node replica costs
+  # tens of minutes of model loading.
+  unreachable_after: 15s
+  lost_after: 60s
+  port_range: [20000, 20999]
+  mgmt_interface: eth0
+
+# Each model needs a source — a weights path or a HuggingFace id — and at least
+# one profile if the control plane is to place it. Without profiles the model is
+# catalogued but never started, which is the router-only mode below.
 models:
   - name: llama-3.3-70b
+    source: meta-llama/Llama-3.3-70B-Instruct
     aliases: [llama]
     fallbacks: [qwen2.5-32b]
     routing_profile: chat
+    scaling: { min: 1, max: 4 }
+    profiles:
+      # Two nodes of four H100s each. A multi-node profile requires a fabric,
+      # and a replica never spans two of them.
+      - name: tp8
+        engine: vllm
+        gpu_class: h100-80g
+        tp: 8
+        nodes: 2
+        gpus_per_node: 4
+        priority: 100
+        weight: 8
+        startup_timeout: 30m
+      # Where a whole node is free, one node is cheaper to schedule and faster
+      # to start than two.
+      - name: tp4-a100
+        engine: vllm
+        gpu_class: a100-40g
+        tp: 4
+        priority: 50
+        weight: 3
+        startup_timeout: 20m
 
   - name: qwen2.5-32b
+    source: Qwen/Qwen2.5-32B-Instruct
     aliases: [qwen]
     routing_profile: batch
+    scaling: { min: 1, max: 2 }
+    profiles:
+      - name: tp2
+        engine: sglang
+        gpu_class: h100-80g
+        tp: 2
+        priority: 100
+        weight: 2
+        startup_timeout: 15m
 
-endpoints:
-  - id: gpu-a-01
-    url: http://gpu-a-01:8000
-    models: [llama-3.3-70b]
-    pool: ib-a/h100-80g
-    weight: 100     # from ctl bench; do not estimate it by hand
-    priority: 100   # higher pools are tried first
-
-  - id: gpu-b-01
-    url: http://gpu-b-01:8000
-    models: [llama-3.3-70b, qwen2.5-32b]
-    pool: ib-b/a100-80g
-    weight: 45
-    priority: 50
+# A static replica list, used only when no model declares a profile. That is the
+# router-only mode for testing against sim-engines; with the control plane
+# running, agents provide the replicas instead.
+#
+# endpoints:
+#   - id: sim-1
+#     url: http://127.0.0.1:8001
+#     models: [llama-3.3-70b]
+#     pool: local
+#     weight: 1

deploy/ha/README.md

@@ -0,0 +1,87 @@
+# Two control planes
+
+llmgw handles the hard half of this itself: only one control plane places work,
+and agents find whichever one is leading. What it does not do is move client
+traffic, because that happens in front of both and llmgw is not in front of
+anything.
+
+This directory is the missing half. It is a pattern rather than a product:
+substitute your own load balancer if you have one.
+
+## What llmgw does on its own
+
+**One leader.** Leadership is a lease in the shared store. A second control
+plane starts, sees the lease, and stands by — it places nothing, evicts
+nothing and drains nothing. Start one by accident and your cluster is
+unharmed.
+
+**Agents follow.** A standby refuses agent streams, so agents move to the
+leader by themselves. Give each agent both addresses:
+
+```yaml
+servers:
+  - "https://ctl-1.internal:7443"
+  - "https://ctl-2.internal:7443"
+```
+
+Measured on a cluster: leadership moved three seconds after the leader was
+stopped, every agent had arrived six seconds later, and every replica was
+adopted rather than restarted — the engines never stopped serving.
+
+## What it does not do
+
+Client traffic. During that same failover the standby served inference
+correctly on its own port while the public address returned 502, because
+nothing in front of the two had moved.
+
+So something has to. The options, in the order most people should consider
+them:
+
+**A load balancer you already run.** Health-check `/ready` on each control
+plane and send traffic to whichever answers. This is the right answer if you
+have one, and it needs nothing else from this directory.
+
+**A floating address.** `keepalived.conf.example` here moves one IP between
+the two machines. Fewer moving parts than a load balancer and no extra hop,
+at the cost of one machine carrying all the traffic.
+
+**DNS.** Works, eventually. A record with a short TTL is a failover measured
+in whatever your resolvers actually honour rather than what you set, which on
+a bad day is minutes. Fine for a cluster people use interactively, wrong for
+one an agent is driving.
+
+## Check `/ready`, not `/health`
+
+This is the one thing to get right, and the obvious choice is the wrong one.
+
+`/health` says the process is up. It says nothing about whether it can serve,
+and a standby is exactly that case: alive, answering, and holding no endpoints
+at all — its reconciler placed nothing and it has no agent streams to adopt
+from. A load balancer checking `/health` would send it half the traffic, and
+half the traffic would be a 503.
+
+`/ready` says this process has somewhere to send a request. It is 503 on a
+standby, 503 on a leader whose replicas have all failed, and 200 when there is
+something to serve — which is the question a load balancer is actually asking.
+Both are open, because a load balancer has no key.
+
+During a failover neither passes for about the length of the lease handover:
+three seconds when the leader is stopped cleanly, up to the lease TTL when it
+is not. That gap is real and there is no way to have it both ways — traffic
+cannot go to a control plane that has nothing to route to.
+
+Not the admin port, for a different reason: it serves pprof, and pprof can
+dump the process heap, where API keys sit in plaintext.
+
+## Both control planes need the same store
+
+Leadership lives in it, and so do keys, budgets and conversations. Two control
+planes on two SQLite files are two clusters that happen to share a name.
+
+```yaml
+auth:
+  store: "@/etc/llmgw/secrets/store-dsn"
+```
+
+See [docs/install.md](../../docs/install.md) for moving an existing
+installation onto PostgreSQL without reissuing every key.

deploy/ha/keepalived.conf.example

@@ -0,0 +1,65 @@
+# A floating address for two control planes.
+#
+# One IP that lives on whichever machine is up. Clients and the reverse proxy
+# in front of llmgw point at the address, not at either host, so a failover is
+# invisible to them.
+#
+# This is not a load balancer: all traffic goes to the holder. That is usually
+# the right trade for a control plane, which serves far less than the nodes
+# it schedules.
+#
+# Copy to /etc/keepalived/keepalived.conf on both machines, changing state,
+# priority and interface.
+
+global_defs {
+    enable_script_security
+    script_user nobody
+}
+
+# The address moves only while llmgw can actually serve, not merely while the
+# machine is powered on.
+#
+# /ready, not /health. /health is 200 on a standby, which is up, answering and
+# holding no endpoints at all — so the address would sit on a host that refuses
+# every request. /ready is 200 only when there is somewhere to send one.
+#
+# Not the admin port either: it serves pprof, which can dump the heap that
+# holds every API key.
+vrrp_script check_llmgw {
+    script "/usr/bin/curl -fsS --max-time 2 -o /dev/null http://127.0.0.1:8080/ready"
+    interval 2
+    timeout 3
+    # Two failures before giving up the address, so one slow moment does not
+    # move it. Two successes before taking it back, so a flapping gateway does
+    # not drag the address with it.
+    fall 2
+    rise 2
+    weight -40
+}
+
+vrrp_instance llmgw {
+    # MASTER on the machine you would rather serve; BACKUP on the other.
+    state MASTER
+    interface eth0
+    # Any number both machines agree on and nothing else on the segment uses.
+    virtual_router_id 51
+    # The higher one wins while both are healthy. The weight above is enough
+    # to invert this when the preferred machine's gateway is not answering.
+    priority 100
+    advert_int 1
+
+    authentication {
+        auth_type PASS
+        # Change this. It stops another host on the segment claiming the
+        # address, which is otherwise a single unicast packet away.
+        auth_pass CHANGEME
+    }
+
+    virtual_ipaddress {
+        10.0.0.150/24
+    }
+
+    track_script {
+        check_llmgw
+    }
+}

deploy/mirror/README.md

@@ -0,0 +1,262 @@
+# A model mirror
+
+One machine that holds the model files, so nodes fetch over your own network
+instead of over the internet.
+
+This is the highest-value piece of infrastructure around llmgw and the cheapest.
+Eight nodes each pulling forty gigabytes from Hugging Face is a day's work, and
+on plenty of corporate networks it is not possible at all. From a mirror on the
+same switch it is minutes.
+
+llmgw needs nothing special for this: a model's `source` is a URL, and the URL
+can be yours.
+
+## What it is
+
+An HTTP server serving a directory. That is the whole design.
+
+```
+/srv/models/
+  qwen2.5-14b-instruct/
+    2026-03/
+      q4_k_m.gguf
+  bge-m3/
+    2026-01/
+      model.safetensors
+```
+
+The date in the path is the point of the layout, not decoration. A model
+repository that changes upstream changes what you are running, and the first
+sign is a prompt that worked last week answering differently this week with
+nobody able to say why. Pin it here and that cannot happen.
+
+## Running it
+
+`llmgw` is the mirror. Same binary as the control plane and the agents, one
+subcommand:
+
+```
+sudo mkdir -p /srv/models && sudo chown llmgw:llmgw /srv/models
+llmgw mirror serve --dir /srv/models --base http://models.internal:8081
+```
+
+There is a unit in `deploy/systemd/llmgw-mirror.service`. Set `--base` to the
+address nodes reach this machine on — not `localhost`, and not a name only this
+host knows itself by. It is the string that ends up in catalog entries, and a
+wrong one makes `mirror unused` report files as unreferenced when they are in
+use. The next step after that list is deletion.
+
+### Two listeners
+
+`--listen :8081` serves the weights. Read-only, unauthenticated, Range requests
+supported so an interrupted fetch on a node resumes rather than restarts. This
+is the path eight nodes pull forty gigabytes from.
+
+`--api-listen :8082` is the management API. Separate on purpose: putting a
+client certificate in front of the byte path would mean re-certifying every
+agent's fetcher in order to change how models are administered.
+
+The API is unauthenticated unless you give it the fleet's CA, and it says so at
+startup rather than leaving you to find out. Anyone who can reach that port can
+start downloads and delete models, so either keep it on a trusted network or
+hand it certificates:
+
+```
+llmgw mirror serve --dir /srv/models --base http://models.internal:8081 \
+  --tls-ca /etc/llmgw/ca.pem \
+  --tls-cert /etc/llmgw/mirror.pem \
+  --tls-key /etc/llmgw/mirror-key.pem
+```
+
+That is the same CA the agents already use, so the certificates come from the
+authority you have:
+
+```
+llmgw ctl cert issue-server mirror --host models.internal --host 10.0.0.146
+llmgw ctl cert issue-client control-plane
+```
+
+`--host` has to cover every address the API is called on, because that is what
+a client verifies against. Both sides are checked — the mirror will not accept a
+caller without a certificate from this CA, the same rule the agent stream
+follows.
+
+### nginx instead
+
+If you already run nginx, it serves a directory perfectly well and you do not
+need `mirror serve` for the bytes:
+
+```nginx
+server {
+    listen 80;
+    server_name models.internal;
+
+    root /srv/models;
+
+    # Read-only. This is deliberate: a mirror that can be written to over HTTP
+    # is a way to replace a model with another one, and the nodes would fetch
+    # it without complaint.
+    location / {
+        autoindex on;
+        # Large files, and the client is a machine with a resumable fetcher.
+        # Sendfile and no buffering keep memory flat regardless of size.
+        sendfile on;
+        tcp_nopush on;
+    }
+}
+```
+
+`autoindex` is there so a person can browse what is available; llmgw does not
+need it.
+
+## Filling it
+
+From the mirror itself:
+
+```
+llmgw mirror add \
+  --model qwen2.5-14b-instruct --version 2026-03 \
+  "https://huggingface.co/<repo>/resolve/<revision>/q4_k_m.gguf"
+```
+
+Use a revision rather than `main` in the source URL. `main` is what moves.
+
+It downloads to a temporary name and renames when complete, so an interrupted
+download never leaves something a catalog could be pointed at, and it prints the
+sha256 when it lands. Compare that against what upstream publishes — Hugging
+Face gives it as `lfs.oid` from
+`https://huggingface.co/api/models/<repo>/tree/<revision>`. A mirror is the one
+place in this system where a corrupt file is copied to every node instead of
+being caught once.
+
+It refuses to overwrite a version that is already there. Add a new one beside
+it: that refusal is what makes the old weights still available when the new ones
+turn out to be worse.
+
+## Driving it from elsewhere
+
+The same fetch over the API, so the control plane — or anything else — can ask
+for a model without anyone logging in:
+
+```
+curl -X POST http://models.internal:8082/v1/models -d '{
+  "source": "https://huggingface.co/<repo>/resolve/<revision>/q4_k_m.gguf",
+  "model": "qwen2.5-14b-instruct",
+  "version": "2026-03"
+}'
+```
+
+The mirror does the downloading. A control plane that pulled the bytes itself
+and pushed them here would carry every model twice over the network, through the
+one machine least able to spare the bandwidth.
+
+It answers `202` with the URL a catalog entry should use, and the bytes are not
+here yet. Poll for them:
+
+| | |
+|---|---|
+| `GET /v1/models` | everything held or arriving, and the base to build URLs from |
+| `POST /v1/models` | fetch this source into model/version |
+| `GET /v1/models/{model}/{version}` | `fetching` with a byte count, `present`, or `failed` with the reason |
+| `DELETE /v1/models/{model}/{version}` | remove a version; refuses while it is downloading |
+| `GET /v1/health` | up *and* able to read its own directory |
+
+A failed fetch stays `failed` with what went wrong, rather than disappearing:
+otherwise the thing waiting on that model waits forever with nothing to say why.
+Asking twice for the same download joins the one already running instead of
+starting a rival writer on the same file.
+
+`DELETE` takes the whole version, never one file of it. A model split across
+several files is useless with one missing, and half a version on disk looks
+present and will not load.
+
+## Retiring a model
+
+```
+llmgw mirror unused     # what no catalog entry refers to
+llmgw mirror remove --model <m> --version <v> --file <f>
+```
+
+`unused` compares against every `source` in the running catalog, so the answer
+is what can go rather than what looks old. It refuses to answer when it cannot
+reach the control plane: an empty catalog and an unreachable one produce the
+same list — everything — and the obvious next step after that list is deletion.
+
+Retire the catalog entry first, let the fleet settle, then remove. A node that
+has to refetch a file you deleted cannot, and the failure arrives later, on the
+node, looking like something else.
+
+## From the control plane's admin page
+
+Configure the mirror on the control plane and the same operations appear at
+`/admin/mirror`, behind the admin session the other admin pages use:
+
+```yaml
+mirror:
+  api: https://models.internal:8082
+  tls:
+    ca: /etc/llmgw/ca.crt
+    cert: /etc/llmgw/control-plane.crt
+    key: /etc/llmgw/control-plane.key
+```
+
+Leave the block out and the page is not mounted at all, which is the right page
+for a deployment with no mirror — not an empty one pointing at nothing.
+
+The page lists what is held, what is arriving with its byte count, and what
+failed with the reason; it starts a fetch, retries a failed one, and removes a
+version. It reloads itself only while something is downloading, because a reload
+at any other moment takes with it whatever is half-typed in the form.
+
+Two things it deliberately will not do. It cannot point the control plane at a
+different mirror — that is configuration, and changing it needs access to the
+host rather than a session. And it does not write the catalog for you: it shows
+the URL an entry should use, and putting it there is still a deliberate act,
+because that is the moment a model starts being served.
+
+The address is a mistake worth naming: `api` is the **management** port, not the
+one nodes fetch bytes from. They are two listeners so this one can be locked
+down without re-certifying every agent's fetcher.
+
+## Pointing llmgw at it
+
+```yaml
+- name: qwen2.5-14b-instruct
+  source: http://models.internal/qwen2.5-14b-instruct/2026-03/q4_k_m.gguf
+  scaling: { min: 2, max: 2 }
+  profiles:
+    - name: gpu
+      engine: vllm
+      gpu_class: h100-80g
+      tp: 1
+```
+
+Each node fetches once, in the background, before its engine starts — and
+`llmgw ctl replicas` shows how far along it is while it does. A killed download
+resumes rather than restarting, and a partial file is never visible under the
+model's name, so an interrupted fetch cannot be handed to an engine.
+
+## What this is not
+
+It is not a cache. Nothing populates it on its own: a model is there because
+somebody asked for it, by hand or over the API, and one nobody asked for is not
+there. That is on purpose. A mirror that fetched on demand would reach the
+internet at the least convenient moment, and in a network where that is blocked
+it would fail in a way that looks like llmgw being broken.
+
+It is also not a replacement for a shared filesystem. If you already run one
+that every node mounts, put the models on it and use a path as the `source` —
+nothing is fetched at all, and llmgw notices the filesystem is shared and will
+not count the same free space once per node.
+
+## When this stops being enough
+
+Somewhere above ten or twenty nodes, one mirror serving a forty gigabyte file
+to all of them at once becomes the bottleneck instead of the fix. At that point
+the options are a filer, a second mirror per rack, or distribution between the
+nodes themselves.
+
+llmgw knows which nodes already hold which weights — it is in `ctl status` as
+"N models on X/Y nodes" — so distributing between them is a feature it is
+positioned to add. It has not been built, because a mirror is enough for
+everyone who has needed it so far.

deploy/monitoring/README.md

@@ -0,0 +1,170 @@
+# Watching a cluster
+
+Prometheus, Alertmanager and Grafana, configured for what llmgw actually
+publishes. Every alert here is written against a series this system emits;
+nothing is aimed at a metric that might exist.
+
+## What it is
+
+One scrape target for the whole fleet.
+
+Agents publish no metrics of their own. The control plane gathers what they
+report — node state, memory, disk, engine queue depth, KV occupancy — and
+republishes all of it, so a single endpoint describes every machine. That is
+worth knowing before going looking for an exporter on each node.
+
+## The port it scrapes
+
+`metrics_listen`, not `admin_listen`.
+
+The admin listener is bound to loopback for one reason: it also serves pprof,
+and pprof can dump the process heap, where API keys sit in plaintext. Metrics
+are the one thing on that listener something else genuinely needs to reach, so
+they get a port of their own instead of becoming an argument for opening the
+other one.
+
+```yaml
+# /etc/llmgw/server.yaml
+metrics_listen: ":9091"
+```
+
+Confirm it carries nothing else:
+
+```
+curl -s -o /dev/null -w '%{http_code}\n' http://<control-plane>:9091/debug/pprof/   # 404
+curl -s -o /dev/null -w '%{http_code}\n' http://<control-plane>:9091/api/v1/nodes   # 404
+```
+
+## Installing
+
+On Ubuntu 24.04 the first two come from the archive; Grafana needs its own
+repository.
+
+```
+apt-get install prometheus prometheus-alertmanager
+
+install -m 0644 prometheus.yml            /etc/prometheus/prometheus.yml
+install -m 0644 llmgw-rules.yml           /etc/prometheus/rules/llmgw.yml
+install -m 0644 alertmanager.yml          /etc/prometheus/alertmanager.yml
+promtool check config /etc/prometheus/prometheus.yml
+systemctl restart prometheus prometheus-alertmanager
+
+install -m 0644 grafana/datasource.yaml   /etc/grafana/provisioning/datasources/llmgw.yaml
+install -m 0644 grafana/dashboards.yaml   /etc/grafana/provisioning/dashboards/llmgw.yaml
+install -m 0644 -o grafana -g grafana grafana/llmgw-overview.json \
+    /var/lib/grafana/dashboards/llmgw.json
+systemctl restart grafana-server
+```
+
+The dashboard is provisioned from a file rather than clicked together, so it
+survives a reinstall and can be reviewed in a diff.
+
+## Where to run it
+
+Not on the control plane. Monitoring that dies with the thing it monitors tells
+you nothing at the moment you need it, and the control plane is also the busiest
+machine in a small cluster.
+
+Anywhere else on the management network will do. The mirror is a reasonable
+host: it has the disk, and serving weights is bursty rather than constant.
+
+## Nothing is sent anywhere
+
+Alertmanager groups and holds alerts and has no receiver.
+
+Where an alert should go — an address, a Slack workspace, a webhook — is a
+decision with credentials attached, and a monitoring system that quietly starts
+mailing people is worse than one that is visibly not wired up. Alerts are
+visible at `:9093` and in Prometheus; add a receiver to `alertmanager.yml` to
+make them travel. `critical` already routes separately so it can be pointed
+somewhere else without touching the rest.
+
+## Check that it fires
+
+An alerting system nobody has seen fire is an assumption.
+
+```
+llmgw ctl cordon <node>     # work already there keeps running
+# ~2 minutes later
+curl -s http://<host>:9093/api/v2/alerts | jq '.[].labels'
+llmgw ctl uncordon <node>   # the alert resolves on its own
+```
+
+Cordoning is the cheapest real test: it changes a metric the rules read, stops
+nothing, and reverses in one command.
+
+## What the alerts assume
+
+**A stopped model publishes no series.** `llmgw_model_replicas_ready` exists
+only for models with a replica placed, so `== 0` cannot fire for one that is
+deliberately off. That is what makes an on-demand catalogue quiet here.
+
+**Saturation of 1 is the healthy limit, not failure.** It is the worse of queue
+depth and KV occupancy, measured by `ctl bench` rather than guessed. The alert
+allows five minutes above it before saying anything.
+
+**Latencies are in seconds.** These thresholds are written for CPU inference. A
+p99 first token of thirty seconds is the alerting point here and would be
+absurd on a GPU; change it when the hardware changes.
+
+**Disk is only reported by nodes that hold weights.** A node with no model
+directory reports nothing rather than zero, and the rule divides free by total
+so a node that reports neither never fires.
+
+## Where the metrics live
+
+On the machine that scrapes them, in Prometheus's own store, and that is the
+right answer at this size. Measured on this cluster: 2,359 series at 41 samples
+a second, which compresses to roughly 7 MB a day — about 3 GB for a year.
+
+```
+# /etc/default/prometheus
+ARGS="--storage.tsdb.retention.time=365d --storage.tsdb.retention.size=20GB"
+```
+
+A year because the questions worth asking of this data are seasonal: what did
+capacity look like last quarter, was it always this slow, how much did the
+fleet grow. Fifteen days answers none of them.
+
+Both limits, not one. The time limit is the intent; the size cap is what stops
+Prometheus becoming the thing that fills the disk if the series count grows
+tenfold after somebody adds a fleet.
+
+No remote write, no object store, no second tier. At two thousand series that
+is infrastructure for a problem this cluster does not have, and every one of
+those has its own failure modes to learn.
+
+### The part that is not about capacity
+
+The disk is not the constraint — 900 GB free against 3 GB a year. What matters
+is what shares it.
+
+On this cluster the model mirror and the metrics database are on the same
+filesystem. A mirror that fills it stops Prometheus writing, so monitoring goes
+quiet at exactly the moment the disk is the problem, and the alert that would
+have said so is the one that died.
+
+Two ways out, in order of how much they cost:
+
+- Watch it, which is now done. `prometheus-node-exporter` arrives with the
+  Prometheus package and was running, unscraped, on this machine — so the disk
+  holding both was watched by nothing. It is a scrape target now, and
+  `MonitoringDiskFilling` fires at fifteen percent rather than five, because
+  the thing being protected is the ability to be told.
+- Separate them: a volume of its own for `/var/lib/prometheus`, or monitoring
+  on its own machine. Worth doing before the mirror holds a few hundred
+  gigabytes; not worth doing today at seven percent used.
+
+## What is not here
+
+No exporter on the llmgw nodes, so no host CPU, load average or network for
+them. The control plane publishes memory, disk and liveness for every node,
+which is what placement decides on; the rest is a `node_exporter` away if
+somebody wants it. The monitoring host itself is scraped, for the reason above.
+
+No probe of the mirror. Its `/v1/health` is JSON rather than Prometheus text,
+and checking it properly means `blackbox_exporter`.
+
+No recording rules. At this size every query in the dashboard runs fast enough
+raw, and a recording rule is a second definition of a number to keep in step
+with the first.

deploy/monitoring/alertmanager.yml

@@ -0,0 +1,41 @@
+# Alertmanager for the llmgw cluster.
+#
+# Nothing is sent anywhere yet, on purpose. Where an alert should go — an
+# address, a Slack workspace, a webhook — is a decision with credentials
+# attached, and a monitoring system that quietly starts mailing people is worse
+# than one that is visibly not wired up. Alerts group and fire here and are
+# visible at :9093 and in Prometheus; add a receiver below to make them travel.
+global:
+  resolve_timeout: 5m
+
+route:
+  # By alert and by what it is about, so five nodes going quiet in the same
+  # network blip arrive as one notification rather than five.
+  group_by: ['alertname', 'node', 'model']
+  group_wait: 30s
+  group_interval: 5m
+  # Long, because a repeat is a reminder and not news. A cluster with one thing
+  # wrong should not produce a message an hour.
+  repeat_interval: 12h
+  receiver: none
+
+  routes:
+    # Critical gets its own branch so it can be pointed somewhere else later
+    # without touching the rest: the difference between "a node is draining"
+    # and "nothing is serving" is who should be woken.
+    - matchers: [severity="critical"]
+      receiver: none
+      group_wait: 10s
+      repeat_interval: 4h
+
+receivers:
+  - name: none
+
+inhibit_rules:
+  # A node that is down explains every replica on it. Reporting both says the
+  # same thing twice and buries whichever is the actual cause.
+  - source_matchers: [alertname="NodeNotReady"]
+    target_matchers: [alertname="EndpointMetricsStale"]
+    equal: ['node']
+  - source_matchers: [alertname="ControlPlaneUnreachable"]
+    target_matchers: [severity="warning"]

deploy/monitoring/grafana/dashboards.yaml

@@ -0,0 +1,11 @@
+apiVersion: 1
+providers:
+  - name: llmgw
+    orgId: 1
+    folder: llmgw
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 30
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards

deploy/monitoring/grafana/datasource.yaml

@@ -0,0 +1,11 @@
+# Provisioned rather than clicked in, so the dashboard below can reference it by
+# a fixed uid and the whole setup survives a reinstall of the machine.
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    uid: llmgw-prom
+    type: prometheus
+    access: proxy
+    url: http://localhost:9090
+    isDefault: true
+    editable: false

deploy/monitoring/grafana/llmgw-overview.json

@@ -0,0 +1,492 @@
+{
+  "uid": "llmgw-overview",
+  "title": "llmgw \u2014 cluster",
+  "tags": [
+    "llmgw"
+  ],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30s",
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "panels": [
+    {
+      "id": 1,
+      "title": "Nodes ready",
+      "type": "stat",
+      "description": "Nodes taking work. A node that is known but draining or cordoned is not counted.",
+      "gridPos": {
+        "x": 0,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "count(llmgw_node_up == 1)",
+          "legendFormat": "ready",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "textMode": "auto",
+        "colorMode": "value"
+      }
+    },
+    {
+      "id": 2,
+      "title": "Replicas serving",
+      "type": "stat",
+      "description": "Across every model. A model deliberately stopped contributes nothing and is not a fault.",
+      "gridPos": {
+        "x": 4,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum(llmgw_model_replicas_ready)",
+          "legendFormat": "ready",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "textMode": "auto",
+        "colorMode": "value"
+      }
+    },
+    {
+      "id": 3,
+      "title": "Models with something serving",
+      "type": "stat",
+      "description": "",
+      "gridPos": {
+        "x": 8,
+        "y": 0,
+        "w": 4,
+        "h": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "count(llmgw_model_replicas_ready > 0)",
+          "legendFormat": "models",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "textMode": "auto",
+        "colorMode": "value"
+      }
+    },
+    {
+      "id": 4,
+      "title": "Requests per second",
+      "type": "timeseries",
+      "description": "Failing is counted separately rather than as a percentage, so a quiet cluster reads as quiet instead of as 0% of nothing.",
+      "gridPos": {
+        "x": 12,
+        "y": 0,
+        "w": 12,
+        "h": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum(rate(llmgw_requests_total[5m]))",
+          "legendFormat": "all",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        },
+        {
+          "refId": "B",
+          "expr": "sum(rate(llmgw_requests_total{status=~\"5..\"}[5m]))",
+          "legendFormat": "failing",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "id": 5,
+      "title": "Replicas ready, by model",
+      "type": "timeseries",
+      "description": "Only models with a replica placed appear. One that is stopped on purpose publishes no series.",
+      "gridPos": {
+        "x": 0,
+        "y": 4,
+        "w": 12,
+        "h": 7
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "llmgw_model_replicas_ready",
+          "legendFormat": "{{model}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      }
+    },
+    {
+      "id": 6,
+      "title": "Pool saturation",
+      "type": "timeseries",
+      "description": "The worse of queue depth and KV occupancy. 1 is the limit measured as healthy, not the point of failure.",
+      "gridPos": {
+        "x": 12,
+        "y": 4,
+        "w": 12,
+        "h": 7
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "llmgw_pool_saturation",
+          "legendFormat": "{{model}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "max": 2,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "id": 7,
+      "title": "Time to first token",
+      "type": "timeseries",
+      "description": "As the router sees it. On CPU these are seconds, not milliseconds; compare against what ctl bench measured.",
+      "gridPos": {
+        "x": 0,
+        "y": 11,
+        "w": 12,
+        "h": 7
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "histogram_quantile(0.5, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))",
+          "legendFormat": "{{model}} p50",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        },
+        {
+          "refId": "B",
+          "expr": "histogram_quantile(0.99, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))",
+          "legendFormat": "{{model}} p99",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "id": 8,
+      "title": "Tokens per second",
+      "type": "timeseries",
+      "description": "",
+      "gridPos": {
+        "x": 12,
+        "y": 11,
+        "w": 12,
+        "h": 7
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (direction) (rate(llmgw_tokens_total[5m]))",
+          "legendFormat": "{{direction}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "id": 9,
+      "title": "Engine queue depth",
+      "type": "timeseries",
+      "description": "Requests the engine has accepted and not started. Rising here is the first sign of saturation.",
+      "gridPos": {
+        "x": 0,
+        "y": 18,
+        "w": 8,
+        "h": 6
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "llmgw_endpoint_waiting",
+          "legendFormat": "{{replica}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      }
+    },
+    {
+      "id": 10,
+      "title": "KV cache occupancy",
+      "type": "timeseries",
+      "description": "",
+      "gridPos": {
+        "x": 8,
+        "y": 18,
+        "w": 8,
+        "h": 6
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "llmgw_endpoint_kv_cache_usage",
+          "legendFormat": "{{replica}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "id": 11,
+      "title": "Model disk free",
+      "type": "timeseries",
+      "description": "The filesystems holding weights. A node that cannot hold the next model is skipped by placement rather than failing.",
+      "gridPos": {
+        "x": 16,
+        "y": 18,
+        "w": 8,
+        "h": 6
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "llmgw_node_disk_free_bytes",
+          "legendFormat": "{{node}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "id": 12,
+      "title": "Seconds since each node last reported",
+      "type": "timeseries",
+      "description": "Heartbeats are every 5s. A line that climbs is an agent going quiet; the control plane stops placing there at 15s.",
+      "gridPos": {
+        "x": 0,
+        "y": 24,
+        "w": 12,
+        "h": 6
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "llmgw_node_last_seen_seconds",
+          "legendFormat": "{{node}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "id": 13,
+      "title": "Replicas by state",
+      "type": "timeseries",
+      "description": "",
+      "gridPos": {
+        "x": 12,
+        "y": 24,
+        "w": 12,
+        "h": 6
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (state) (llmgw_replicas)",
+          "legendFormat": "{{state}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      }
+    }
+  ]
+}

deploy/monitoring/llmgw-rules.yml

@@ -0,0 +1,155 @@
+# Alerts for llmgw. Every expression here is against a series this cluster
+# actually publishes; nothing is written against a metric that might exist.
+groups:
+  - name: llmgw-fleet
+    rules:
+      - alert: ControlPlaneUnreachable
+        expr: up{job="llmgw"} == 0
+        for: 1m
+        labels: { severity: critical }
+        annotations:
+          summary: "The control plane is not answering"
+          description: >-
+            Nothing has scraped 10.0.0.140:9091 for a minute. Engines already
+            running keep serving — the router is in the same process, so if
+            this is down, so is the gateway.
+
+      - alert: NodeNotReady
+        expr: llmgw_node_up == 0
+        for: 2m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.node }} is {{ $labels.state }}"
+          description: >-
+            The node is known but not taking work. Two minutes, because a node
+            restarting its agent is briefly not ready and is not a problem.
+
+      - alert: AgentGoneQuiet
+        expr: llmgw_node_last_seen_seconds > 60
+        for: 1m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.node }} has not reported for {{ $value | humanizeDuration }}"
+          description: >-
+            Heartbeats are every 5s. A minute of silence is the network or the
+            agent, and the control plane stages expiry at 15s unreachable and
+            60s lost — so this fires around the point it stops placing there.
+
+      - alert: ModelHasNothingServing
+        expr: llmgw_model_replicas_ready == 0
+        for: 10m
+        labels: { severity: critical }
+        annotations:
+          summary: "{{ $labels.model }} has no replica ready"
+          description: >-
+            The model is placed and nothing is serving it. Ten minutes, because
+            a large model on CPU legitimately takes minutes to load. A model
+            that is deliberately stopped publishes no series at all, so this
+            cannot fire for one nobody asked for.
+
+      - alert: PoolSaturated
+        expr: llmgw_pool_saturation > 1
+        for: 5m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.model }} is over its healthy limit"
+          description: >-
+            Saturation is the worse of queue depth and KV occupancy; 1 is the
+            limit measured as healthy, not the point of failure. Sustained above
+            it means requests are queueing — add a replica or lower the load.
+
+      - alert: EndpointBreakerOpen
+        expr: llmgw_endpoint_breaker_open == 1
+        for: 2m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.replica }} has been taken out of rotation"
+          description: >-
+            The router stopped sending to this replica after repeated failures.
+            Traffic is going elsewhere; if it is the only replica, the model is
+            down.
+
+      - alert: EndpointMetricsStale
+        expr: llmgw_endpoint_metrics_stale == 1
+        for: 5m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.replica }} is being routed to blind"
+          description: >-
+            Its queue depth and KV usage are old, so the scheduler is choosing
+            with stale numbers. The replica may be serving perfectly well; the
+            decisions about it are the problem.
+
+      - alert: NodeDiskFilling
+        expr: llmgw_node_disk_free_bytes / llmgw_node_disk_total_bytes < 0.10
+        for: 15m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.node }} has {{ $value | humanizePercentage }} of its model disk free"
+          description: >-
+            Weights are tens of gigabytes. A node that cannot hold the next one
+            is skipped by placement rather than failing, so this is quiet until
+            somebody wonders why a model will not land there.
+
+  - name: monitoring-host
+    rules:
+      - alert: MonitoringDiskFilling
+        expr: >-
+          node_filesystem_avail_bytes{job="monitoring-host",fstype!~"tmpfs|overlay|squashfs"}
+          / node_filesystem_size_bytes{job="monitoring-host",fstype!~"tmpfs|overlay|squashfs"} < 0.15
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.mountpoint }} on the monitoring host is {{ $value | humanizePercentage }} free"
+          description: >-
+            This filesystem holds the model mirror and the metrics database at
+            once. Filling it stops new weights arriving and stops Prometheus
+            writing — so monitoring goes quiet at the moment it is most needed,
+            which is why this fires at fifteen percent rather than five.
+
+      - alert: MonitoringDiskAlmostFull
+        expr: >-
+          node_filesystem_avail_bytes{job="monitoring-host",fstype!~"tmpfs|overlay|squashfs"}
+          / node_filesystem_size_bytes{job="monitoring-host",fstype!~"tmpfs|overlay|squashfs"} < 0.05
+        for: 5m
+        labels: { severity: critical }
+        annotations:
+          summary: "{{ $labels.mountpoint }} on the monitoring host has {{ $value | humanizePercentage }} left"
+
+  - name: llmgw-traffic
+    rules:
+      # These need traffic through the gateway to have any series at all. A
+      # counter with no observations publishes nothing, so a quiet cluster
+      # simply has no data here rather than a rule that is always false.
+      - alert: HighErrorRate
+        expr: >-
+          sum by (model) (rate(llmgw_requests_total{status=~"5.."}[5m]))
+          / sum by (model) (rate(llmgw_requests_total[5m])) > 0.05
+        for: 5m
+        labels: { severity: critical }
+        annotations:
+          summary: "{{ $labels.model }} is failing {{ $value | humanizePercentage }} of requests"
+
+      - alert: SlowFirstToken
+        expr: >-
+          histogram_quantile(0.99,
+            sum by (model, le) (rate(llmgw_ttft_seconds_bucket[10m]))) > 30
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.model }} takes {{ $value | humanizeDuration }} to first token at p99"
+          description: >-
+            On CPU this is expected under load and is why the threshold is
+            thirty seconds rather than one. Compare with what ctl bench measured
+            before treating it as a fault.
+
+      - alert: RequestsBeingRejected
+        expr: sum by (band) (rate(llmgw_admission_rejected_total[5m])) > 0
+        for: 5m
+        labels: { severity: warning }
+        annotations:
+          summary: "Admission is turning away {{ $labels.band }} traffic"
+          description: >-
+            Working as designed when the cluster is full: the lowest band is
+            held first. Worth knowing about because the callers being refused
+            do not know why.

deploy/monitoring/prometheus.yml

@@ -0,0 +1,42 @@
+# Prometheus for the llmgw cluster.
+#
+# One scrape target for the whole fleet. Agents publish nothing of their own —
+# the control plane gathers what they report and republishes it, so node health,
+# disk and endpoint state all arrive here from 10.0.0.140.
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+  external_labels:
+    cluster: mynodes
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ['localhost:9093']
+
+rule_files:
+  - /etc/prometheus/rules/llmgw.yml
+
+scrape_configs:
+  # The control plane's dedicated metrics port. Not its admin listener: that one
+  # is loopback because it also serves pprof, and pprof can dump the heap where
+  # API keys sit in plaintext.
+  - job_name: llmgw
+    static_configs:
+      - targets: ['10.0.0.140:9091']
+        labels:
+          instance: control-plane
+
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:9090']
+
+  # The machine this runs on. It came with the Prometheus package and was not
+  # being scraped, which left the disk holding both the model mirror and the
+  # metrics themselves watched by nothing — so a mirror that filled it would
+  # take monitoring down and the alert that would have warned did not exist.
+  - job_name: monitoring-host
+    static_configs:
+      - targets: ['localhost:9100']
+        labels:
+          instance: monitoring

deploy/systemd/README.md

@@ -0,0 +1,34 @@
+# systemd units
+
+Two roles, two units. Copy the one you need to `/etc/systemd/system/`, put the
+matching configuration in `/etc/llmgw/`, and `systemctl enable --now` it.
+
+## The control plane — `llmgw-server.service`
+
+Runs unprivileged. It needs to read its own TLS key, so that file must belong to
+the `llmgw` user and nothing else:
+
+```
+useradd --system --home-dir /var/lib/llmgw --shell /usr/sbin/nologin llmgw
+chown llmgw:llmgw /etc/llmgw/server-key.pem
+chmod 0400 /etc/llmgw/server-key.pem
+```
+
+`StateDirectory=llmgw` creates `/var/lib/llmgw` with the right owner, which is
+where the store belongs. `ProtectSystem=strict` makes the rest of the filesystem
+read-only, so anything else it writes to has to be named explicitly.
+
+## The agent — `llmgw-agent.service`
+
+Runs as root, because it launches engines and needs device access.
+
+`KillMode=process` is the line that matters and the reason this file exists
+rather than being generated. Without it, systemd kills the whole control group
+on stop — including the engines, which hold a loaded model and are serving
+traffic. Engines are meant to outlive an agent restart: the router talks to them
+directly, and the agent adopts them again when it comes back. Taking them down
+with the agent turns a thirty-second upgrade into a full model reload on every
+node.
+
+systemd will log `Found left-over process ... in control group while starting
+unit. Ignoring.` on the next start. That is the mechanism working, not a fault.

deploy/systemd/llmgw-agent.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=llmgw node agent
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+ExecStart=/usr/local/bin/llmgw agent --config /etc/llmgw/agent.yaml
+# Root, because the agent launches engines and needs device access. Engines
+# outlive a restart of this unit and are adopted again on the next start, so
+# KillMode must not take the whole process group with it.
+KillMode=process
+Restart=on-failure
+RestartSec=2s
+StateDirectory=llmgw-agent
+
+[Install]
+WantedBy=multi-user.target

deploy/systemd/llmgw-mirror.service

@@ -0,0 +1,29 @@
+[Unit]
+Description=llmgw model mirror
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+# --base is what a catalog entry will say, so it has to be the address nodes
+# reach this machine on — not localhost, and not the name only this host knows
+# itself by. Getting it wrong makes `mirror unused` report files as unreferenced
+# when they are in use, and the next step after that list is deletion.
+ExecStart=/usr/local/bin/llmgw mirror serve \
+    --dir /srv/models \
+    --listen :8081 \
+    --api-listen :8082 \
+    --base http://models.internal:8081
+User=llmgw
+Group=llmgw
+Restart=on-failure
+RestartSec=2s
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+# The one writable path. The mirror serves reads to the whole fleet and writes
+# only what its own API was asked to fetch, so nothing else needs to be.
+ReadWritePaths=/srv/models
+
+[Install]
+WantedBy=multi-user.target

deploy/systemd/llmgw-server.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=llmgw control plane
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+ExecStart=/usr/local/bin/llmgw server --config /etc/llmgw/server.yaml
+User=llmgw
+Group=llmgw
+Restart=on-failure
+RestartSec=2s
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+PrivateTmp=true
+StateDirectory=llmgw
+
+[Install]
+WantedBy=multi-user.target

docs/choosing-models.md

@@ -0,0 +1,149 @@
+# Choosing what to run
+
+llmgw serves whatever you give it and has no opinion about which models those
+are. This page has an opinion, because "we have GPUs, now what" is the question
+that stalls most of these projects — and the usual first answer, one large
+model, is the wrong shape.
+
+Model releases move faster than documentation. Treat the families named here as
+a starting point and check what is current before committing; the reasoning is
+what should outlive the names.
+
+## Three jobs, not one model
+
+The requests a company actually makes fall into three groups with different
+needs, and no single model serves all three well.
+
+**Interactive** — chat, autocomplete, short questions. Most of your traffic.
+Here latency beats quality: a person waits two seconds and does not wait
+twenty. A 7–14B class model, always resident.
+
+**Heavy** — long document analysis, code generation, anything that needs
+reasoning. Quality beats latency, and a queue is acceptable. A 30–70B class
+model, loaded on demand:
+
+```yaml
+scaling: { min: 0, max: 1, idle_timeout: 30m }
+```
+
+llmgw starts it when somebody asks and stops it when nobody has, so it costs
+nothing while it is not in use. The first request waits for the load, and the
+client is told that rather than refused.
+
+**Embeddings** — and this is the one that gets forgotten. Most "let us chat
+with our documents" projects are retrieval projects, and retrieval runs on an
+embedding model. It is small, it is cheap, and it carries the highest request
+volume of the three: every document chunk once, every query forever. llmgw
+routes `/v1/embeddings` like any other endpoint.
+
+A deployment with the first two and not the third can answer questions from
+what the model already knew. It cannot answer questions about your company.
+
+## Swedish and English
+
+The first thing to know is that this combination is harder than either alone.
+
+Open models are strongest in English by a wide margin. Swedish is served —
+better than most languages its size, because the Nordic languages appear in
+multilingual training sets — but the gap is real, and it widens as models get
+smaller. **A 7B model that feels fine in English can feel noticeably worse in
+Swedish**, which means the interactive tier may need to be larger than an
+English-only deployment would choose. Test before deciding; do not assume the
+English benchmark transfers.
+
+Two things matter more than the model's headline multilingual score:
+
+**Cross-lingual retrieval.** A Swedish company writes internal documents in
+Swedish and reads technical documentation in English. Somebody will ask a
+question in Swedish whose answer is in an English document. An embedding model
+that handles both languages separately may still fail to connect them, and it
+fails silently — the model gives a confident answer from the wrong source, and
+nobody notices because the answer reads well. Test this explicitly: ask in one
+language, check that documents in the other are retrieved.
+
+**Swedish compounds.** *Kundtjänstmedarbetare* is one token-hostile word where
+English would use three. This hurts keyword search more than dense retrieval,
+so if you run hybrid search, expect the lexical half to underperform on Swedish
+and weight accordingly.
+
+## How to actually decide
+
+Benchmarks will not settle this. Take twenty real documents and twenty real
+questions from the company — in both languages, including at least a few where
+the question and the answer are in different languages — and compare candidates
+on those. An afternoon of this is worth more than a week of reading leaderboards,
+and it is the only way to find the failure modes that matter to you.
+
+Do the same for the embedding model, separately. A good generation model on top
+of bad retrieval produces fluent, confident, wrong answers, which is the worst
+failure this stack has: it does not look like a failure.
+
+## How many replicas
+
+Not "how many employees". **Concurrency**, which is far lower than headcount
+because people spend most of their time reading rather than waiting on a model.
+Two hundred people generate perhaps five to fifteen concurrent requests at
+peak, for interactive use.
+
+vLLM batches continuously, so one replica on a capable card serves many
+concurrent requests. Which means the replica count is usually decided by
+**redundancy rather than throughput**:
+
+| Tier | Replicas | Why |
+|---|---|---|
+| Embeddings | 2 | Not for capacity. If retrieval stops, every document feature stops. |
+| Interactive | 2 | So a node can be rebooted during working hours. |
+| Heavy | 0–1, on demand | Loaded when asked for, stopped when idle. |
+
+Then measure rather than guess:
+
+```
+llmgw ctl bench <model>
+```
+
+It climbs concurrency until the SLO breaks and reports the last level that met
+it. Peak throughput arrives long after latency has become unusable, so
+throughput alone overstates what the hardware can serve.
+
+## Where the weights live
+
+The single highest-value piece of infrastructure here is **a local mirror**: one
+internal HTTP server holding the model files. [deploy/mirror/](../deploy/mirror/)
+has the nginx config and the directory layout. Hugging Face is slow, rate
+limited, and blocked outright on plenty of corporate networks. Eight nodes each
+pulling forty gigabytes from the internet is a day; from a mirror on the same
+switch it is minutes. Point a model's `source` at it and llmgw fetches from
+there.
+
+**Pin the version.** Put it in the path — `/models/qwen2.5-14b/2026-03/q4_k_m.gguf`
+— so a repository that changes upstream cannot change what you are running.
+Without this, a prompt that worked last week gives a different answer this week
+and nobody can explain why.
+
+**Quantisation.** For llama.cpp, `q4_k_m` is the practical default: the quality
+difference from `q8_0` is smaller than most people expect and the memory
+difference is a factor of two. For vLLM on modern cards, FP8 or AWQ. Run
+`llmgw ctl plan <model>` to see whether a given precision fits before
+downloading anything.
+
+**Disk.** Budget three to five times what you first estimate. Several
+quantisations, several versions, and partial downloads all accumulate.
+`llmgw ctl nodes` shows free space per node once `model_dirs` is configured,
+and `llmgw ctl weights` reclaims files llmgw downloaded and no model needs —
+never files you placed yourself.
+
+Under ten nodes, let each keep its own copy; it is simpler and the disk is
+cheap. Above that, consider a shared filesystem — llmgw notices one and will
+not count the same free terabyte once per node.
+
+## Before anyone starts using it
+
+Set budgets and limits on day one, not later:
+
+```
+llmgw keys limits --scope project --id <project> --rpm 60 --budget 50000000
+```
+
+The reason is not technical. A resource that appears free and unlimited becomes,
+three months later, a cost nobody can account for — and introducing limits at
+that point is a much harder conversation than having had them from the start.

docs/data.md

@@ -0,0 +1,133 @@
+# What llmgw stores, and what leaves the machine
+
+Written for the review that happens before a gateway is allowed near anyone's
+documents. It describes what the software does. Mapping that onto an
+obligation — GDPR, a sector rule, an internal policy — is work for whoever
+holds the obligation; this page exists so that work starts from facts rather
+than from a vendor's summary.
+
+Everything below is checkable in the source, and the places to look are named.
+
+## The short version
+
+By default llmgw stores **no prompt or completion text anywhere**. It stores
+who made a request, when, how many tokens, and what it cost.
+
+Two features change that, and both are off until somebody turns them on for a
+named project, and neither can be turned on without also saying for how long.
+
+Nothing on the serving path leaves the cluster.
+
+## What is stored, exactly
+
+**Usage records**, one per request: the key, the project, the model asked for,
+the model that answered, prompt and completion token counts, cost, HTTP status
+and latency. No message content. The row is in `internal/store/store.go` as
+`Usage` and there is no field for text in it.
+
+**API keys**, as SHA-256 hashes. The key itself is shown once, at creation, and
+never stored — so a leaked database hands over no working credential, and
+neither llmgw nor an administrator can recover a key somebody lost. A prefix is
+kept so a key can be identified in a listing without revealing it.
+
+**People and tenancy**: accounts, the organisation/team/project tree, and the
+limits attached to it. Passwords are hashed.
+
+**Sessions** for the browser pages, as tokens.
+
+**Conversations and agent notes** — only for a project that asked. See below.
+
+## What is not stored
+
+Prompts. Completions. Anything the model was shown or produced, unless the
+project turned on conversations or notes.
+
+The chat page in llmgw's own web interface keeps the conversation in a
+JavaScript variable in the browser. Refresh it and the conversation is gone,
+because it was never anywhere else.
+
+## Conversations and notes, when they are turned on
+
+Two features keep text, and both work the same way:
+
+- **Conversations** (`x-llmgw-thread`) let a client send only its new turn while
+  llmgw replays the history — including to a different model than the one the
+  conversation started on.
+- **Agent notes** (`/v1/memory`, and MCP tools) let an agent keep something for
+  itself across conversations.
+
+Both are **off for every project** until an operator runs `llmgw keys threads`
+or `llmgw keys memory` for that project by name. A request carrying a thread id
+against a project that has not is refused with an explanation, not quietly
+served without its history.
+
+Both **require a retention window**. There is no way to turn either on without
+saying how many hours, because a store with no expiry is a transcript of
+everything a team ever typed, kept forever because nobody chose a number. The
+window is measured from the last turn, so an active conversation does not
+expire underneath it, and a background sweep removes what has passed it.
+
+Both are **scoped to a project**, and the scope is part of every lookup rather
+than a check afterwards. Reading a conversation that belongs to another project
+returns the same answer as reading one that does not exist, so a leaked id
+cannot be used to discover what exists elsewhere.
+
+`DELETE /v1/threads/{id}` removes a conversation and its messages immediately.
+It is a client endpoint, not an administrative one: the person who said the
+thing can unsay it without asking somebody else to read it first.
+
+Turning retention off stops new storage, and existing conversations go at the
+next hourly sweep.
+
+## What leaves the machine
+
+**Nothing on the serving path.** A request arrives, is routed to an engine on
+your own hardware, and the answer comes back. The router talks to engines, the
+engines talk to nothing, the agents talk to the control plane. All of it is
+inside the cluster.
+
+Two things can reach outside, both started by an operator rather than by a
+request:
+
+**Fetching weights.** When a model's `source` is a URL, the agent downloads it.
+Point that at an internal mirror and nothing leaves at all — which is also the
+faster arrangement, because eight nodes pulling forty gigabytes from the
+internet is a day's work and from a local mirror it is minutes. A `source` that
+is a path on disk is never fetched.
+
+**`llmgw ctl plan`.** Reads a model's architecture from huggingface.co to work
+out whether it fits your hardware. An operator command run at planning time,
+not part of serving, and not run by anything automatically.
+
+There is no telemetry, no licence check, and no call home. The binary is
+static, built with `CGO_ENABLED=0`, and runs in a network with no route to the
+internet as long as the weights are already on disk.
+
+## What the software cannot do for you
+
+**The admin port serves pprof**, and pprof can dump the process heap — where
+API keys sit in plaintext while they are being checked. Bind it to loopback and
+reach it over SSH. The default does this; a deployment that changes it should
+know why.
+
+**The store holds everything.** With conversations turned on it holds text as
+well as accounting, so where it lives, who can read it and how it is backed up
+are decisions with the same weight as the ones above. The PostgreSQL DSN
+belongs in a file readable only by the user llmgw runs as, which is why the
+configuration takes `@/path/to/file` rather than the address itself.
+
+**Model weights are yours.** llmgw reclaims only files it downloaded, and it
+identifies those by a name it generated. Anything you put on a node it will not
+touch, whatever the catalog says.
+
+## Where to look
+
+| Claim | Where |
+|---|---|
+| Usage rows carry no text | `internal/store/store.go`, type `Usage` |
+| Keys are stored hashed | `internal/store/keys.go` |
+| Conversations are off by default | `internal/store/threads.go`, `ThreadRetention` |
+| Retention is required | `llmgw keys threads --help` |
+| Scope is part of the lookup | `internal/store/threads.go`, `Thread` |
+| Deletion is a client endpoint | `internal/router/threads.go`, `handleDeleteThread` |
+| Weight fetching is the only serving-path egress | `internal/agent/weights/fetch.go` |

docs/node-onboarding.md

@@ -0,0 +1,230 @@
+# Adding a node
+
+Written for whoever runs the cluster.
+
+## The short version
+
+Where one person holds both the hypervisor and the certificate authority —
+which is the common case — one command does the whole thing in about
+45 seconds:
+
+```
+llmgw-add-node \
+  --name llmgw-node05 --ip 10.0.0.145/24 \
+  --server https://10.0.0.140:7443 \
+  --cores 8 --memory 32768 --disk 60 \
+  --llamacpp-bin /opt/llama.cpp/llama-server
+```
+
+The VMID defaults to the last octet of the address, so `qm list` and
+`llmgw ctl nodes` stay in the same order. `--provision-only` and
+`--onboard-only` split it back into the two halves below when the two kinds
+of access belong to different people.
+
+The rest of this page is what that command does, and what it still does not.
+
+## The two halves
+
+Two scripts, because the two halves of onboarding are done by different people
+with different access: one needs the hypervisor, the other needs the
+certificate authority.
+
+## 1. The machine
+
+On the Proxmox host:
+
+```
+llmgw-provision-node --vmid 144 --name llmgw-node04 --ip 10.0.0.144/24
+```
+
+Name it after the node id. **The VM name, the hostname, the node id and the
+certificate's common name are all the same string.** They do not have to be, and
+every time they are not, somebody spends an afternoon working out which of the
+four a message is talking about.
+
+It imports a cloud image and lets cloud-init do the rest, which is why a node
+takes a minute rather than an afternoon. It refuses to touch an existing VMID:
+adopting a machine is not a decision a provisioning script should make.
+
+Releases carry static `linux/amd64` and `linux/arm64` binaries. Download one and
+pass it with `--binary` rather than copying whatever happens to be on the
+machine you are running onboarding from — otherwise whoever runs the script
+quietly becomes the build server, and the fleet ends up running whatever was in
+their working tree.
+
+## 2. The identity
+
+On the host that holds the certificate authority — this is the part that cannot
+be delegated, because it issues the credential by which the control plane
+believes the node is who it says it is:
+
+```
+llmgw-onboard-node \
+  --node llmgw-node04 \
+  --addr 10.0.0.144 \
+  --server https://10.0.0.140:7443 \
+  --fabric ib-b
+```
+
+It issues a client certificate whose common name is the node id, installs it
+along with the CA root, trusts that root system-wide so other tooling on the
+machine stops treating llmgw's certificates as strangers, installs the binary
+and the unit, and starts the agent. The node appears in `llmgw ctl nodes` within
+a few seconds.
+
+If it does not, the control plane's log says why. A certificate whose common
+name does not match the claimed node id is refused by design — that check is
+what makes a node's identity something it proves rather than something it picks.
+
+**It needs ssh from the CA host to the node.** Provisioning copies that host's
+`authorized_keys` into the new machine, so a node built by the script above
+already satisfies it; an older machine may need the key added.
+
+It waits for ssh rather than failing on it, and learns the host key of a machine
+it has not seen (`accept-new`, which still refuses a *changed* key — the risk
+being guarded against is a substituted machine, not a new one). Both are needed
+to run the halves back to back: for the first minute of its life a new machine
+is still regenerating host keys and restarting sshd, so it refuses connections
+it accepted a moment earlier.
+
+It also checks that a configured engine actually runs, which is cheap here and
+expensive later — see below.
+
+## The CPU model matters, and its default is wrong
+
+Nodes are provisioned with `--cpu host`. Proxmox's default, `kvm64`, hides
+AVX, AVX2, FMA and even SSE4.2 from the guest, and llama.cpp then loads its
+scalar backend instead of the one built for the host's instruction set.
+Measured on node05, an 8-core VM on a Xeon E5-2680 v4, same weights, same
+threads, the only change being the CPU model:
+
+| Qwen2.5-Coder  | kvm64 tok/s | host tok/s |
+| -------------- | ----------: | ---------: |
+| 1.5B q4_K_M    |        5.80 |      39.09 |
+| 3B q4_K_M      |        2.90 |      20.51 |
+| 7B q4_K_M      |        1.33 |       9.88 |
+
+Prompt processing is worse still: 7.6 tokens a second against 143.8, a factor
+of nineteen.
+
+It is worth calling out because nothing looks broken. The node registers, the
+engine starts, replies are correct — every number is simply small, and the
+obvious conclusion is that CPU inference is hopeless rather than that the VM
+cannot multiply efficiently.
+
+The trade is live migration: a VM pinned to the host CPU cannot move to a
+machine with a different one. For a node whose job is arithmetic that is the
+right way round, and `--cpu` overrides it where it is not.
+
+## What onboarding does not do
+
+It gives a machine an identity and an agent. It does **not** install inference
+engines or model weights, and a node missing either looks healthy in
+`llmgw ctl nodes` right up until a replica is placed on it and the shard dies
+with "executable file not found".
+
+Onboarding does check that an engine you point it at can be executed, and says
+so loudly when it cannot. That check exists because the failure is invisible
+otherwise: `llama-server` on node05 was present, executable, and could not start
+for want of `libgomp1`, and the node reported `ready` in `llmgw ctl nodes` the
+whole time. Being told at onboarding beats finding out when a replica is placed
+and the reason is three logs away.
+
+So the node is not finished until:
+
+- the engines its catalog profiles name are installed (`vllm`, `sglang`,
+  `llama-server`), at the paths the agent configuration gives
+- the weights are on disk, at the `source` each model declares
+
+Both are deliberately outside this script. Engines are a platform decision with
+versions and drivers attached, and distributing weights is its own problem —
+hundreds of gigabytes that want a shared filesystem or a cache, not an scp from
+a laptop. llmgw will fetch weights for a model whose `source` is a URL, one
+copy per node, which covers the simple case; anything larger stays the
+operator's arrangement, and llmgw is content to find the files already there.
+
+The `sim` engine needs neither, which is what makes it useful for exercising
+everything else.
+
+## Checking an engine after an upgrade
+
+Engines rename and remove flags between versions, and when that happens the
+failure arrives minutes into a model load, in the engine's own error message, on
+a node nobody was watching.
+
+```
+llmgw ctl engine-check vllm
+llmgw ctl engine-check llamacpp --help-output "$(llama-server --help 2>&1)"
+```
+
+It builds the widest command llmgw would ever produce — a multi-node one, since
+a flag that only appears there is exactly the one nobody notices has gone — and
+checks every flag against the engine's own `--help`. No GPU required: a CLI
+prints its help on any machine.
+
+Some programs do not give up their whole help for `--help`. vLLM prints a
+summary of section names and expects `--help=all`; checking against the summary
+reports every flag as missing, confidently and wrongly. `engine-check` knows
+that one. If you add an engine, check that what you get back is the full list.
+
+vLLM also refuses to do anything at all on a machine with no accelerator,
+including print its help, unless `VLLM_TARGET_DEVICE` says what to assume.
+`engine-check` sets it.
+
+Give it the **whole** help. Truncating llama.cpp's 725 lines to 400 reports five
+flags as missing that the running cluster uses; the command warns when the text
+it was given does not document `--help` itself, which is the cheapest sign it
+was cut short.
+
+## Fabrics
+
+`--fabric` is how a node says which InfiniBand fabric it is attached to, and
+there is no way around stating it. OpenSM's default subnet prefix is `fe80::` on
+every fabric, so a matching prefix proves nothing and a wrong guess produces a
+replica that starts, passes health checks, and then hangs on its first
+all-reduce.
+
+Omit it for a node with no fabric. Such a node is its own island: it can host
+single-node replicas and will never be given a multi-node one.
+
+Two adapters per fabric is the default; `--fabric ib-b:mlx5_2,mlx5_3` names them
+explicitly. An adapter must never appear under two fabrics — the agent refuses
+that configuration, because the consequence is the hang above.
+
+## Where the driver is
+
+```
+--nvidia-smi /opt/nvidia/bin/nvidia-smi
+--ib-root /sys/class/infiniband
+```
+
+Both default to the usual places. Containerized driver installs routinely put
+`nvidia-smi` somewhere else, and an agent that cannot find it registers with no
+GPUs — which looks exactly like a node that has none.
+
+## Removing a node
+
+```
+llmgw ctl drain llmgw-node04     # move its replicas off, one at a time
+llmgw ctl nodes                  # wait until it holds none
+```
+
+Then stop the agent. Draining brings each replacement up before stopping the
+original, which is the difference between a rolling move and an outage.
+
+The node's certificate stays valid until it expires. There is no revocation list
+— if a machine is decommissioned rather than retired, reissue the authority or
+accept that a certificate naming a node that no longer exists can still connect
+and claim to be it.
+
+## Testing without hardware
+
+Every part of this works on machines with no GPUs and no InfiniBand. Point
+`--nvidia-smi` at a script that prints the CSV discovery asks for, point
+`--ib-root` at a sysfs-shaped directory, and use `engine: sim` in the catalog.
+
+The simulated engine rendezvouses for real: rank 0 binds the dist-init port and
+does not serve until every worker has joined, and workers serve nothing. That
+exercises the multi-node launch sequence, the fabric constraint, and failure and
+rebuild — which is most of what goes wrong — without exercising NCCL, which is
+the rest.

docs/onboarding.md

@@ -0,0 +1,319 @@
+# Onboarding a user
+
+Written for whoever operates an llmgw deployment: the person with a shell on the
+control plane machine.
+
+There are two ways a person gets a key. Pick one and put it in the config.
+
+| | Self-service | Operator-issued |
+|---|---|---|
+| Config | `auth.signup_open: true` | `auth.signup_open: false` |
+| They do | sign up, wait, then make their own keys | wait for a link |
+| You do | approve once | issue every key |
+| Good for | more than a handful of people | a few, or machines |
+
+Self-service does not mean open. A signup produces an account that can do
+nothing at all; **approval is what makes it work**, and approval needs a shell
+here rather than a session cookie, because it is the step that decides who gets
+to spend the cluster.
+
+Everything below runs as the user the server runs as, so the files it creates
+stay readable by it:
+
+```
+sudo -u llmgw llmgw <command> --store /var/lib/llmgw/llmgw.db
+```
+
+## Self-service
+
+Turn on the form:
+
+```yaml
+auth:
+  enabled: true
+  store: /var/lib/llmgw/llmgw.db
+  signup_open: true
+```
+
+The landing page then links to it, and the rest is:
+
+```
+llmgw users list --pending --store …
+ID                    EMAIL              NAME   STATE    PROJECT  SIGNED UP
+usr-k4mq…             ayse@example.com   Ayşe   pending  -        4m12s ago
+
+llmgw users approve usr-k4mq… --project default --by ilker --store …
+```
+
+That is the whole operator loop. From then on they sign in at `/login`, and
+`/profile` is where they create keys, see when each was last used, and revoke the
+ones they are done with. A key is shown once on the page that creates it and
+never again.
+
+**This gateway sends no email.** There is no "your account was approved" message
+and no password reset, so tell people what to expect: they sign up, they tell
+you, you approve, they sign in. If that loop is too manual for your size, it is
+the thing to automate — not the approval step.
+
+To close an account:
+
+```
+llmgw users suspend usr-k4mq… --store …
+```
+
+which closes their sessions immediately. Their **API keys keep working** — those
+are separate credentials, and revoking them is a separate step:
+
+```
+llmgw keys list --project default --store …
+llmgw keys revoke key-… --store …
+```
+
+That separation is deliberate: a suspended person's running jobs should not all
+fail at once without someone deciding they should.
+
+## Operator-issued
+
+With `signup_open: false` there is no form, and you issue keys yourself. This is
+also the path for things that are not people — CI, a notebook server, a script.
+
+### Decide where the key belongs
+
+Keys hang off a project, projects off a team, teams off an organization. Limits
+can be set at any of those levels and compose downward by taking the **most
+restrictive** value — a key cannot raise what its team set, and a team cannot
+raise what its organization set. That is the only rule under which a budget
+means anything.
+
+If you have no structure yet, one of each is fine:
+
+```
+llmgw keys init --store … --org mynodes --team default --project default
+```
+
+It is idempotent: running it again renames rather than duplicating, so it can
+live in whatever provisions the machine.
+
+Split it when you want separate limits or separate reporting, not before.
+
+### Issue the key as a one-time link
+
+```
+llmgw keys create --store … \
+  --project default \
+  --band gold \
+  --base-url https://llmgw.example \
+  --claim \
+  "ayse-laptop"
+
+https://llmgw.example/claim/3mKq…
+```
+
+Send the link, not the key.
+
+A key pasted into a chat window stays in that chat window forever, searchable by
+everyone who joins the channel later. A link works once and then expires, so the
+worst case is someone opening it before the intended person — which you find out
+about, because they will tell you the link is dead.
+
+The name is for you, not the user: it appears in `keys list` and is how you will
+know which key to revoke.
+
+**Why the link is safe to store.** The key is sealed with the link's own token,
+and the token is never written down — only its hash is. A database leak yields
+two useless hashes and a blob nothing in the database can open.
+
+Without `--claim` the key is printed to stdout instead, for when you are
+provisioning something that is not a person:
+
+```
+llmgw keys create --store … --project ci "github-runner" > /run/secrets/llmgw-key
+```
+
+## Limits
+
+Limits are optional. A key with none is unlimited, which is the right default on
+a private cluster and the wrong one the moment the gateway faces anything else.
+
+```
+# Everyone in this project
+llmgw keys limits --store … --scope project --id default \
+  --rpm 60 --tpm 200000
+
+# One key, tighter
+llmgw keys limits --store … --scope key --id key-mpmip77twpt7qaxc \
+  --rpm 10
+
+# Restrict which models
+llmgw keys limits --store … --scope project --id default \
+  --models qwen-0.5b,llama-3.3-70b
+```
+
+An **absent** model list permits everything. An **empty** one permits nothing —
+`--models ""` is a way to suspend a project without revoking its keys.
+
+Allowlists compose by intersection: a key cannot add a model its team withheld.
+A model's canonical name and any of its aliases name the same thing, so either
+form works.
+
+Budgets are counted in micro-units, and what a unit means is yours to decide.
+Self-hosted models have no per-token price, so it can be GPU-seconds, a notional
+rate, or 1 000 000 everywhere — which turns a budget into a plain token quota.
+Models are priced in the server's `auth.pricing` section; a model priced at zero
+is exempt from budget checks, which is what lets a key with nothing left still be
+served a cheap fallback.
+
+```
+llmgw keys limits --store … --scope project --id default \
+  --budget 50000000 --period month        # 50 units a month
+```
+
+## The landing page
+
+Someone who reaches the gateway without a key sees the landing page. Tell it who
+to send them to, or it can only shrug:
+
+```yaml
+landing:
+  contact: "platform@example.com"   # or an https:// URL, or "#platform on Slack"
+  notice: "Internal use only."
+```
+
+An address becomes a mailto link, an `http(s)` URL stays a link, anything else is
+shown as text. Other schemes are never linked — configuration is not always
+written by whoever runs the gateway, and a `javascript:` href on a page served to
+every visitor is a short path to a stored XSS.
+
+## What the user sees
+
+They open the link, press the button, and get the key once. The page shows them
+the base URL and a working `curl`, and points at two endpoints:
+
+- `GET /v1/models` — what this key may use
+- `GET /me` — its limits and what it has spent
+
+`/me` is there so nobody has to ask you "why was I rate limited" or "how much
+have I used". It is outside `/v1` because it is not part of the OpenAI API.
+
+## Rotation and revocation
+
+```
+llmgw keys list --store … --project default
+```
+
+`LAST USED` is written in batches rather than per request, so it lags by up to a
+minute. It is the column that tells you whether a key is still in use, which is
+what makes rotation safe:
+
+**To rotate**, issue the new key, let both work while the user switches, watch
+`LAST USED` on the old one stop moving, then revoke it.
+
+```
+llmgw keys revoke --store … key-mpmip77twpt7qaxc
+```
+
+Revocation is a soft delete: the key stops working, the row stays, and its usage
+history keeps its subject. A running server keeps accepting a revoked key until
+its cache expires — `auth.cache_ttl`, 30 seconds by default. That is the price of
+keeping the database off the request path, and it is the number to lower if you
+need revocation to bite faster.
+
+## Passwords
+
+People change their own from `/profile`. It asks for the current one — a session
+is what someone gets from a borrowed laptop, and the password is what they do not
+have — and it signs every other browser out, which is the point if they are
+changing it because something went wrong.
+
+There is no self-service reset, because this gateway sends no email and a reset
+link with nowhere to send it is not a reset. When someone loses theirs:
+
+```
+llmgw users reset-password usr-k4mq… --password "a temporary passphrase" --store …
+```
+
+Tell them out of band and have them change it from their profile. Their sessions
+are closed — if they lost the password, the reason might be that someone else has
+it. Their API keys keep working; those are separate credentials.
+
+## Trying a model in the browser
+
+`/chat` is a playground for anyone signed in. It exists so somebody can see
+whether a model works without setting up a client, and — more often — so they
+can tell whether a problem is the gateway or their code.
+
+**The browser never holds a key.** The page carries a session; the server turns
+that into the same identity an API key would produce, resolved from the person's
+project, and the request goes down the ordinary path: admission, model
+allowlist, rate limits, budget, accounting. Anything else would make the page a
+hole in every limit the rest of the gateway enforces.
+
+Usage from the page is attributed to `session:<user-id>` rather than a key, so
+it is billed to the right project and visibly did not come from a credential
+somebody holds. One turn is capped at 512 tokens: a playground should not be the
+thing that ties up a replica for ten minutes, and anyone who wants a longer
+answer has an API key for that.
+
+Nothing is stored. The conversation lives in the page and is gone when it is
+closed.
+
+## Seeing the cluster
+
+The dashboard — nodes, replicas, models, why something did not get placed, usage
+per key — is served twice, and the difference matters.
+
+On the **admin listener** (`127.0.0.1:9090` by default) it needs no
+authentication: having a shell on the machine is the credential.
+
+```
+ssh -L 9090:127.0.0.1:9090 <control-plane>   # then http://127.0.0.1:9090/
+```
+
+That listener also serves pprof, which can dump the process heap — where API
+keys sit in plaintext. **Never expose that port.** It is bound to loopback for
+that reason.
+
+On the **client-facing port** the same page is at `/admin`, behind a session and
+an admin flag, so it can be reached from a browser without the port carrying
+pprof along with it:
+
+```
+llmgw users promote usr-k4mq… --store …
+```
+
+Someone without the flag gets a 404 rather than a 403, because 403 confirms the
+page exists.
+
+## Managing accounts from the browser
+
+`/admin/users` lists who is waiting and who is active. Approving (into a project)
+and suspending are there, because that is the work an operator does constantly
+and it should not need a terminal.
+
+Three things are deliberately not there:
+
+| | Why |
+|---|---|
+| `llmgw users promote` | An admin who can make admins turns a stolen session into permanent access |
+| `llmgw users reset-password` | Setting someone's password is taking over their account |
+| `llmgw keys create` | Issuing a credential on another person's behalf |
+
+Each of those turns a stolen admin cookie into more than it was, and each is
+rare enough that a shell is no hardship. Cordon, drain and everything else that
+touches the cluster are also CLI-only for the same reason: the dashboard shows
+the cluster, it does not steer it.
+
+Every administrative action is logged with who did it, including the ones that
+were refused.
+
+## What this does not do
+
+- **No email.** No approval notice, no self-service password reset, no address
+  verification. The loop is manual on purpose at this size; if it stops being
+  the right size, that is the thing to automate.
+- **No SSO.** Accounts are local, with local passwords.
+- **No roles.** There is one flag, `is_admin`, and it grants looking at the
+  dashboard. Everything else an administrator does is the CLI, and having a
+  shell here is what makes someone one.
+- **Rate limits are per gateway instance.** With two instances each counts its own,
+  so the effective limit doubles. v1 is single-instance by design.

docs/quickstart.md

@@ -0,0 +1,375 @@
+# Quickstart: one machine, one model, ten minutes
+
+Written for someone who wants to see llmgw answer a request on a single
+workstation before reading anything else. No systemd, no `sudo`. For a real
+cluster, see [install.md](install.md).
+
+What you end up with: an OpenAI-compatible endpoint on `localhost:8080` in front
+of a real model served by llama.cpp.
+
+## Two ways to run
+
+Steps 1 to 4 use the simple one, **router-only**: you start the engines yourself
+and tell the router where they are with `--endpoint`. No certificates, no node
+agent.
+
+Step 5 uses the **control plane and node agent**, the same architecture as a real
+cluster on one machine: the agent starts and supervises the engine, and the
+control plane places the model.
+
+## 1. Build
+
+```
+git clone https://gitea.mynodes.xyz/ilker/llmgw && cd llmgw
+make build
+make test
+./bin/llmgw --version
+```
+
+`make build` writes `./bin/llmgw`. Go 1.26 or later is the only requirement.
+
+## 2. Check the path with a fake engine
+
+Before involving a GPU, prove the router works. `sim-engine` is an
+OpenAI-compatible fake that streams placeholder words.
+
+```
+./bin/llmgw sim-engine --listen :8001 --model llama &
+./bin/llmgw server --endpoint e1=http://127.0.0.1:8001=llama &
+
+curl -N localhost:8080/v1/chat/completions \
+  -H 'content-type: application/json' \
+  -H 'x-session-id: demo' \
+  -d '{"model":"llama","messages":[{"role":"user","content":"hello"}],"stream":true}'
+```
+
+Chunks of `alpha beta gamma ...` ending in `data: [DONE]` mean the router is
+fine. Stop both before continuing (see [Stopping](#stopping)).
+
+## 3. Serve a real model
+
+Two processes, two ports. Clients only ever talk to the router; the router
+knows where the engine is because you tell it with `--endpoint`:
+
+```
+curl :8080  ->  llmgw router :8080  ->  llama serve :8082  ->  model
+ (client)        (the gateway)           (the engine)
+```
+
+Requests go to 8080. Port 8082 is the engine's own, and you only touch it to
+check that the engine is up.
+
+llmgw does not install engines or weights. You need:
+
+- a `llama-server` build from llama.cpp (CUDA build if you want the GPU), and
+- a `.gguf` file.
+
+Check the card first with `nvidia-smi`. An 8 GB card holds a 1B model at bf16
+(2.4 GB) or a 9B model at `q4_k_m` (5.4 GB).
+
+Start the engine on its own port. The model name given to `--alias` is the name
+clients will use:
+
+```
+llama serve \
+  -m /path/to/Llama-3.2-1B-Instruct-BF16.gguf \
+  --host 0.0.0.0 --port 8082 -ngl 99 --alias llama
+```
+
+`-ngl 99` offloads every layer to the GPU. Confirm it is up and knows the model:
+
+```
+curl -s localhost:8082/v1/models
+```
+
+Then point the router at it. The format is `id=URL=model`:
+
+```
+./bin/llmgw server --endpoint e1=http://127.0.0.1:8082=llama
+```
+
+## 4. Ask it something
+
+Streaming:
+
+```
+curl -N localhost:8080/v1/chat/completions \
+  -H 'content-type: application/json' \
+  -H 'x-session-id: demo' \
+  -d '{"model":"llama","messages":[{"role":"user","content":"hello"}],"stream":true}'
+```
+
+One reply at once:
+
+```
+curl -s localhost:8080/v1/chat/completions \
+  -H 'content-type: application/json' \
+  -d '{"model":"llama","messages":[{"role":"user","content":"hello"}],"max_tokens":100}'
+```
+
+Also useful: `curl -s localhost:8080/v1/models` and
+`curl -s localhost:8080/health`. `llmgw ctl` does not work in this mode: it
+talks to the control plane's admin API, which router-only mode does not have,
+so `ctl status` answers `404`.
+
+On the machine this was written on, a 1B bf16 model on an RTX 4070 laptop GPU
+answered at roughly 85 tokens per second.
+
+## Several replicas of one model
+
+Give the router more than one `--endpoint` with the same model name and it
+spreads requests across them. `x-session-id` keeps a conversation on the same
+replica so its cache is reused:
+
+```
+./bin/llmgw server \
+  --endpoint e1=http://127.0.0.1:8082=llama \
+  --endpoint e2=http://127.0.0.1:8083=llama
+```
+
+## Stopping
+
+```
+pkill -f "llmgw server"
+pkill -f "llmgw sim-engine"
+```
+
+Stop the engine the way you started it (Ctrl-C in its terminal).
+
+## What goes wrong
+
+**`listening on :8080: bind: address already in use`.** An earlier router is
+still running. `ss -ltnp | grep 8080` names the process; stop it first. This is
+the usual result of moving from step 2 to step 3 without stopping the fake
+engine's router.
+
+**`/v1/models` on the engine returns an empty list.** The engine was started
+without `-m`, so it is a router with nothing to load. Restart it with a model.
+
+**`error: nothing to serve`.** `llmgw server` with no `--endpoint` and no model
+carrying `profiles` has nothing to do. In router-only mode, always pass at least
+one `--endpoint`.
+
+**Requests fail with a model-not-found error.** The name in the request must
+match the name after the last `=` in `--endpoint`, and it must be a name the
+engine actually serves (its `--alias`).
+
+**Answers are poor.** A 1B model is a plumbing test, not a chat assistant. Swap
+in a larger `.gguf`; nothing else changes.
+
+## 5. Let llmgw run the engine (control plane and node agent)
+
+Steps 2 and 3 start the engine by hand. Here a **node agent** starts and
+supervises llama.cpp for you, and a control plane decides where the model runs.
+Still one machine, still no `sudo`. Stop everything from steps 2 and 3 first:
+the GPU has to be free and port 8080 unused.
+
+```
+curl :8080  ->  llmgw server (control plane + router)  <-  llmgw agent (this node)
+                                                              |
+                                                       starts llama-server :20000
+```
+
+The agent dials the control plane over mutual TLS, so this needs a small private
+certificate authority. Everything goes in a scratch directory, `devcluster/`,
+which is in `.gitignore`.
+
+### Certificates
+
+```
+mkdir -p devcluster/pki
+./bin/llmgw ctl cert init-ca --dir devcluster/pki
+./bin/llmgw ctl cert issue-server llmgw-ctl --host 127.0.0.1 --host localhost \
+  --dir devcluster/pki --out devcluster
+./bin/llmgw ctl cert issue-client llmgw-node01 --dir devcluster/pki --out devcluster
+```
+
+The name given to `issue-client` becomes the certificate's common name and must
+equal `node_id` in the agent config. The control plane refuses an agent whose
+claimed id differs from its certificate.
+
+### The engine wrapper
+
+The agent starts the engine as `<bin> --model ... --port ...`. A `llama` binary
+that wants `llama serve ...` refuses every flag, so put a wrapper in front of it:
+
+```
+cat > devcluster/llama-server <<'EOF'
+#!/bin/sh
+exec /path/to/llama serve "$@"
+EOF
+chmod +x devcluster/llama-server
+
+./bin/llmgw ctl engine-check llamacpp --bin devcluster/llama-server
+```
+
+`engine-check` should end with `every flag llmgw builds is accepted`. If your
+build already ships a `llama-server` binary, point `bin` at that and skip the
+wrapper.
+
+### Control plane config: `devcluster/server.yaml`
+
+```yaml
+listen: "127.0.0.1:8080"
+admin_listen: "127.0.0.1:9090"
+agent_listen: "127.0.0.1:7443"
+
+agent_tls:
+  ca: devcluster/pki/ca.pem
+  cert: devcluster/server.pem
+  key: devcluster/server-key.pem
+
+models:
+  - name: llama
+    source: /path/to/Llama-3.2-1B-Instruct-BF16.gguf
+    routing_profile: chat
+    context_length: 8192
+    scaling: { min: 1, max: 1 }
+    profiles:
+      - name: gpu
+        engine: llamacpp
+        gpu_class: 4070-8g
+        weight: 1
+        priority: 100
+        startup_timeout: 5m
+```
+
+Three things in there are easy to get wrong:
+
+- **The model must have at least one profile.** That is what switches the control
+  plane on. With `models: []` the server starts as a plain router and exits with
+  `nothing to serve`.
+- **`gpu_class` decides CPU or GPU.** Without it the replica is placed in the CPU
+  pool and runs on the CPU. Get the class name from `./bin/llmgw agent inventory`
+  (the `CLASS` column).
+- **There is no `auth:` block.** Leave it out for a local trial; with auth on,
+  requests need an API key.
+
+### Agent config: `devcluster/agent.yaml`
+
+```yaml
+node_id: llmgw-node01
+server: "https://127.0.0.1:7443"
+mgmt_addr: "127.0.0.1"
+
+tls:
+  ca: devcluster/pki/ca.pem
+  cert: devcluster/llmgw-node01.pem
+  key: devcluster/llmgw-node01-key.pem
+
+engines:
+  llamacpp:
+    bin: /absolute/path/to/devcluster/llama-server
+
+supervisor:
+  work_dir: devcluster/agent-work
+```
+
+### Run it
+
+Two terminals:
+
+```
+./bin/llmgw server --config devcluster/server.yaml
+./bin/llmgw agent  --config devcluster/agent.yaml
+```
+
+The model is not placed at once. The control plane waits about 35 seconds for the
+fleet to settle after it starts, then places the replica, and the agent then
+loads the weights. Watch it with:
+
+```
+./bin/llmgw ctl nodes
+./bin/llmgw ctl models
+./bin/llmgw ctl replicas
+./bin/llmgw ctl explain-placement llama
+```
+
+`STATUS ready` in `ctl models` means it is serving. `nvidia-smi` should now show
+the model in GPU memory. The same curl from step 4 works against port 8080.
+
+`ctl` works here, unlike in router-only mode, because there is a control plane
+for it to talk to.
+
+Stop it with Ctrl-C in each terminal. The agent leaves the engine running when
+the agent itself stops, so if you restart the agent and the GPU stays full,
+`pgrep -af "llama serve"` will show the engine still up.
+
+## 6. Memory: conversations and notes
+
+Off unless you ask. llmgw otherwise stores nothing anyone typed. On a single
+machine with no API keys, one block in the server config turns on two things:
+
+```yaml
+threads:
+  store: devcluster/data/threads.db
+  retain_hours: 24
+```
+
+Restart the server after adding it. `retain_hours` is required: everything kept
+expires that long after it was last written.
+
+Everything kept this way sits in one shared pool with no owner, and anyone who
+can reach the port and knows an id can read it. That is fine on one machine and
+wrong on a shared network. With auth on, threads and notes are per project
+instead and the block is ignored; see [install.md](install.md).
+
+### Conversations (threads)
+
+Send the same `x-llmgw-thread` id and only the new turn. llmgw puts the stored
+history in front of it:
+
+```
+curl -s localhost:8080/v1/chat/completions \
+  -H 'content-type: application/json' -H 'x-llmgw-thread: demo' \
+  -d '{"model":"llama","messages":[{"role":"user","content":"My name is Deniz."}]}'
+
+curl -s localhost:8080/v1/chat/completions \
+  -H 'content-type: application/json' -H 'x-llmgw-thread: demo' \
+  -d '{"model":"llama","messages":[{"role":"user","content":"What is my name?"}]}'
+
+curl -X DELETE localhost:8080/v1/threads/demo     # forget it
+```
+
+Without the block, a request carrying `x-llmgw-thread` is refused with
+`this gateway keeps no conversations` rather than answered without its history.
+
+### Notes an agent keeps for itself
+
+Notes outlive any one conversation. Each is a key, some text and optional tags,
+kept per agent; name the agent with `x-llmgw-agent` (default `default`), so two
+agents using the same key do not read each other's notes.
+
+```
+curl -X PUT localhost:8080/v1/memory/favorite-color \
+  -H 'content-type: application/json' \
+  -d '{"text":"The user likes dark green.","tags":["preference"]}'
+
+curl 'localhost:8080/v1/memory?q=green'          # search by words
+curl 'localhost:8080/v1/memory?tag=preference'   # or by tag
+curl localhost:8080/v1/memory/favorite-color
+curl -X DELETE localhost:8080/v1/memory/favorite-color
+```
+
+The same notes are offered to an agent as tools over MCP at
+`http://localhost:8080/mcp`: `remember`, `recall` and `forget`. Add that URL to
+your agent's MCP server list (HTTP transport; how you set request headers such
+as `x-llmgw-agent` depends on the client). llmgw does not put these tools into
+chat requests on its own, so an agent that is not given the server never sees
+memory.
+
+Limits: a key is up to 200 bytes, a note's text up to 16 KiB, at most 16 tags,
+1000 notes per agent. Tags are lowercase letters, digits, `-` and `_`.
+
+## Where this stops
+
+Router-only mode (steps 2 to 4) has no authentication, no placement and no model
+management: you start and restart engines yourself. Step 5 adds placement, model
+management and a node agent, and can add authentication as `install.md`
+describes. For a real cluster, with several nodes, systemd and a persistent
+certificate authority, follow [install.md](install.md).
+
+One caveat found while writing this page: `install.md` recommends starting with
+an empty `models: []` and adding models later with `ctl model add`. The control
+plane only turns on when the config declares a model with `profiles`, so that
+first start does not produce a control plane.

go.mod

@@ -3,7 +3,10 @@ module gitea.mynodes.xyz/ilker/llmgw
 go 1.26.3
 
 require (
+	connectrpc.com/connect v1.21.0
+	github.com/NVIDIA/go-nvml v0.13.4-0
 	github.com/cespare/xxhash/v2 v2.3.0
+	github.com/jackc/pgx/v5 v5.11.0
 	github.com/knadh/koanf/parsers/yaml v1.1.1
 	github.com/knadh/koanf/providers/env/v2 v2.0.1
 	github.com/knadh/koanf/providers/file v1.2.1
@@ -12,21 +15,36 @@ require (
 	github.com/prometheus/client_model v0.6.3
 	github.com/prometheus/common v0.71.0
 	github.com/spf13/cobra v1.10.2
+	golang.org/x/crypto v0.57.0
+	golang.org/x/net v0.59.0
+	golang.org/x/sync v0.23.0
+	google.golang.org/protobuf v1.36.12
+	modernc.org/sqlite v1.59.0
 )
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/knadh/koanf/maps v0.1.2 // indirect
+	github.com/mattn/go-isatty v0.0.24 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
 	github.com/prometheus/procfs v0.21.1 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
 	go.yaml.in/yaml/v3 v3.0.5 // indirect
-	golang.org/x/sync v0.23.0 // indirect
-	golang.org/x/sys v0.47.0 // indirect
-	google.golang.org/protobuf v1.36.12 // indirect
+	golang.org/x/sys v0.48.0 // indirect
+	golang.org/x/text v0.42.0 // indirect
+	modernc.org/libc v1.75.7 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.12.1 // indirect
 )

go.sum

@@ -1,16 +1,37 @@
+connectrpc.com/connect v1.21.0 h1:LhqSJt7jHf5NJBo9Jq/t/9FjcYAideif0mg+qe2jCUs=
+connectrpc.com/connect v1.21.0/go.mod h1:A2ygJrukXwWy32vkCAAHNVguZrqZ+jeZ9rGRnGR4dN4=
+github.com/NVIDIA/go-nvml v0.13.4-0 h1:o3jp9u2x1R9ShFE3v+Aesp55XOSIQFMJz/VGNUcJaNE=
+github.com/NVIDIA/go-nvml v0.13.4-0/go.mod h1:id63qwpoDWpFXwnwM6psDCSqW4BmNu6mWpr4YeQtPGo=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
+github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
 github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo=
@@ -25,12 +46,17 @@ github.com/knadh/koanf/v2 v2.3.6 h1:JoQPSJmvS4aP0xNc8xMDr5tcrkSEInL23/Il7pITAKo=
 github.com/knadh/koanf/v2 v2.3.6/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
+github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
 github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
 github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
@@ -39,11 +65,16 @@ github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3
 github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=
 github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
 github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
 github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
 github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
 github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
 github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@@ -53,10 +84,49 @@ go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
 go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
+golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
+golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
+golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
+golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
+golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
+golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
 golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
 golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
-golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
-golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
+golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
+golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
+golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
+golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
 google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
 google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8=
+modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
+modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w=
+modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I=
+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
+modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew=
+modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
+modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
+modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A=
+modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

internal/account/account_test.go

@@ -0,0 +1,396 @@
+package account
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+func newStore(t *testing.T) *store.DB {
+	t.Helper()
+	s, err := store.OpenSQLite(context.Background(), filepath.Join(t.TempDir(), "a.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = s.Close() })
+
+	ctx := context.Background()
+	for _, err := range []error{
+		s.PutOrg(ctx, store.Org{ID: "org", Name: "Org"}),
+		s.PutTeam(ctx, store.Team{ID: "team", OrgID: "org", Name: "Team"}),
+		s.PutProject(ctx, store.Project{ID: "proj", TeamID: "team", Name: "Proj"}),
+	} {
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	return s
+}
+
+func newService(t *testing.T) (*Service, *store.DB) {
+	t.Helper()
+	s := newStore(t)
+	return New(s, DefaultConfig(), slog.New(slog.DiscardHandler)), s
+}
+
+const goodPassword = "correct horse battery staple"
+
+func TestSignupLeavesAnAccountWaiting(t *testing.T) {
+	// A public signup form that produced a working account would be a way in.
+	svc, st := newService(t)
+	ctx := context.Background()
+
+	if err := svc.SignUp(ctx, "Ayse@Example.COM", "Ayşe", goodPassword); err != nil {
+		t.Fatal(err)
+	}
+	u, err := st.UserByEmail(ctx, "ayse@example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if u.State != store.UserPending {
+		t.Errorf("a new account is %q", u.State)
+	}
+	if u.Email != "ayse@example.com" {
+		t.Errorf("the address was not normalized: %q", u.Email)
+	}
+
+	if _, _, err := svc.SignIn(ctx, "ayse@example.com", goodPassword); !errors.Is(err, ErrNotApproved) {
+		t.Errorf("a pending account signed in: %v", err)
+	}
+}
+
+func TestSignupDoesNotRevealExistingAccounts(t *testing.T) {
+	// A form that says "this address already has an account" is an account
+	// enumerator anyone can run, and the person signing up learns nothing they
+	// need: either way they wait for approval.
+	svc, _ := newService(t)
+	ctx := context.Background()
+
+	if err := svc.SignUp(ctx, "ayse@example.com", "A", goodPassword); err != nil {
+		t.Fatal(err)
+	}
+	if err := svc.SignUp(ctx, "ayse@example.com", "Someone Else", goodPassword); err != nil {
+		t.Errorf("the second signup reported %v, which confirms the address is taken", err)
+	}
+}
+
+func TestApprovalIsWhatMakesAnAccountWork(t *testing.T) {
+	svc, st := newService(t)
+	ctx := context.Background()
+	must(t, svc.SignUp(ctx, "ayse@example.com", "A", goodPassword))
+
+	u, _ := st.UserByEmail(ctx, "ayse@example.com")
+	must(t, st.ApproveUser(ctx, u.ID, "proj", "ilker", time.Now()))
+
+	token, csrf, err := svc.SignIn(ctx, "ayse@example.com", goodPassword)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if token == "" || csrf == "" {
+		t.Fatal("sign-in produced no session")
+	}
+	_, got, err := svc.Resolve(ctx, token)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.ProjectID != "proj" {
+		t.Errorf("the approved account landed in project %q", got.ProjectID)
+	}
+}
+
+func TestApprovalNeedsAProject(t *testing.T) {
+	// An approved account with nowhere to put its keys passes every check and
+	// then fails at the first key it tries to make.
+	_, st := newStoreAndUser(t)
+	if err := st.ApproveUser(context.Background(), "usr-1", "", "ilker", time.Now()); err == nil {
+		t.Error("an account was approved into no project")
+	}
+}
+
+func newStoreAndUser(t *testing.T) (*Service, *store.DB) {
+	t.Helper()
+	svc, st := newService(t)
+	hash, _ := HashPassword(goodPassword)
+	must(t, st.CreateUser(context.Background(), store.User{
+		ID: "usr-1", Email: "ayse@example.com", Name: "A",
+		PasswordHash: hash, State: store.UserPending, CreatedAt: time.Now(),
+	}))
+	return svc, st
+}
+
+func TestSuspensionTakesEffectOnOpenSessions(t *testing.T) {
+	// Leaving existing sessions open would make suspension advisory.
+	svc, st := newStoreAndUser(t)
+	ctx := context.Background()
+	must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
+
+	token, _, err := svc.SignIn(ctx, "ayse@example.com", goodPassword)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := svc.Resolve(ctx, token); err != nil {
+		t.Fatal(err)
+	}
+
+	must(t, st.SetUserState(ctx, "usr-1", store.UserSuspended))
+	if _, _, err := svc.Resolve(ctx, token); !errors.Is(err, store.ErrNoSession) {
+		t.Errorf("a suspended account kept its session: %v", err)
+	}
+}
+
+func TestSignOutEndsTheSession(t *testing.T) {
+	svc, st := newStoreAndUser(t)
+	ctx := context.Background()
+	must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
+
+	token, _, _ := svc.SignIn(ctx, "ayse@example.com", goodPassword)
+	must(t, svc.SignOut(ctx, token))
+	if _, _, err := svc.Resolve(ctx, token); !errors.Is(err, store.ErrNoSession) {
+		t.Errorf("the session survived sign-out: %v", err)
+	}
+}
+
+func TestAWrongPasswordAndAMissingAccountAreIndistinguishable(t *testing.T) {
+	svc, st := newStoreAndUser(t)
+	ctx := context.Background()
+	must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
+
+	_, _, wrong := svc.SignIn(ctx, "ayse@example.com", "wrong password here")
+	_, _, missing := svc.SignIn(ctx, "nobody@example.com", "wrong password here")
+	if !errors.Is(wrong, ErrBadCredentials) || !errors.Is(missing, ErrBadCredentials) {
+		t.Fatalf("wrong=%v missing=%v", wrong, missing)
+	}
+}
+
+func TestGuessingIsBounded(t *testing.T) {
+	// Without this a public login form is an open brute-force target.
+	svc, st := newStoreAndUser(t)
+	ctx := context.Background()
+	must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
+
+	cfg := DefaultConfig()
+	cfg.MaxAttempts = 3
+	svc = New(st, cfg, slog.New(slog.DiscardHandler))
+
+	for i := range 3 {
+		if _, _, err := svc.SignIn(ctx, "ayse@example.com", "nope"); !errors.Is(err, ErrBadCredentials) {
+			t.Fatalf("attempt %d: %v", i, err)
+		}
+	}
+	if _, _, err := svc.SignIn(ctx, "ayse@example.com", "nope"); !errors.Is(err, ErrTooManyTries) {
+		t.Errorf("guessing was not bounded: %v", err)
+	}
+	// And a correct password is refused too while the window is open, or the
+	// limit would be trivially bypassed by interleaving.
+	if _, _, err := svc.SignIn(ctx, "ayse@example.com", goodPassword); !errors.Is(err, ErrTooManyTries) {
+		t.Errorf("the window did not apply to a correct password: %v", err)
+	}
+}
+
+func TestASuccessfulSignInClearsTheWindow(t *testing.T) {
+	// Otherwise a person who mistypes their password a few times is locked out
+	// for the rest of the window even after getting it right.
+	st := newStore(t)
+	hash, _ := HashPassword(goodPassword)
+	ctx := context.Background()
+	must(t, st.CreateUser(ctx, store.User{
+		ID: "usr-1", Email: "a@example.com", Name: "A",
+		PasswordHash: hash, State: store.UserPending, CreatedAt: time.Now(),
+	}))
+	must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
+
+	cfg := DefaultConfig()
+	cfg.MaxAttempts = 3
+	svc := New(st, cfg, slog.New(slog.DiscardHandler))
+
+	for range 2 {
+		_, _, _ = svc.SignIn(ctx, "a@example.com", "nope")
+	}
+	if _, _, err := svc.SignIn(ctx, "a@example.com", goodPassword); err != nil {
+		t.Fatal(err)
+	}
+	for range 3 {
+		if _, _, err := svc.SignIn(ctx, "a@example.com", "nope"); errors.Is(err, ErrTooManyTries) {
+			t.Fatal("the window was not cleared by a successful sign-in")
+		}
+	}
+}
+
+func TestCSRFTokensAreCheckedAgainstTheSession(t *testing.T) {
+	svc, st := newStoreAndUser(t)
+	ctx := context.Background()
+	must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
+
+	token, csrf, _ := svc.SignIn(ctx, "ayse@example.com", goodPassword)
+	sess, _, _ := svc.Resolve(ctx, token)
+
+	if !svc.CheckCSRF(sess, csrf) {
+		t.Error("the session's own token was rejected")
+	}
+	for _, bad := range []string{"", "wrong", csrf + "x", csrf[:len(csrf)-1]} {
+		if svc.CheckCSRF(sess, bad) {
+			t.Errorf("%q was accepted as a csrf token", bad)
+		}
+	}
+	// A token from another session must not work in this one.
+	other, otherCSRF, _ := svc.SignIn(ctx, "ayse@example.com", goodPassword)
+	_ = other
+	if svc.CheckCSRF(sess, otherCSRF) {
+		t.Error("another session's token was accepted")
+	}
+}
+
+func TestPasswordsAreHashedNotStored(t *testing.T) {
+	svc, st := newService(t)
+	ctx := context.Background()
+	must(t, svc.SignUp(ctx, "ayse@example.com", "A", goodPassword))
+
+	u, _ := st.UserByEmail(ctx, "ayse@example.com")
+	if strings.Contains(u.PasswordHash, goodPassword) {
+		t.Fatal("the password is in the database")
+	}
+	if !strings.HasPrefix(u.PasswordHash, "$argon2id$") {
+		t.Errorf("hash format: %q", u.PasswordHash)
+	}
+	ok, err := VerifyPassword(goodPassword, u.PasswordHash)
+	if err != nil || !ok {
+		t.Errorf("the hash does not verify: %v %v", ok, err)
+	}
+	if ok, _ := VerifyPassword("something else entirely", u.PasswordHash); ok {
+		t.Error("a wrong password verified")
+	}
+}
+
+func TestTwoHashesOfOnePasswordDiffer(t *testing.T) {
+	// A salt that did not vary would make the table a rainbow-table lookup.
+	a, _ := HashPassword(goodPassword)
+	b, _ := HashPassword(goodPassword)
+	if a == b {
+		t.Fatal("hashing is not salted")
+	}
+}
+
+func TestShortPasswordsAreRefused(t *testing.T) {
+	if _, err := HashPassword("short"); !errors.Is(err, ErrWeakPassword) {
+		t.Errorf("a five-character password was accepted: %v", err)
+	}
+	svc, _ := newService(t)
+	if err := svc.SignUp(context.Background(), "a@example.com", "A", "short"); !errors.Is(err, ErrWeakPassword) {
+		t.Errorf("signup accepted it: %v", err)
+	}
+}
+
+func TestMalformedHashesAreRejectedRatherThanMatched(t *testing.T) {
+	for _, bad := range []string{"", "plaintext", "$argon2id$", "$bcrypt$v=19$m=1,t=1,p=1$aa$bb"} {
+		if ok, err := VerifyPassword(goodPassword, bad); ok || err == nil {
+			t.Errorf("VerifyPassword(%q) = %v, %v", bad, ok, err)
+		}
+	}
+}
+
+func TestObviousNonAddressesAreRefused(t *testing.T) {
+	svc, _ := newService(t)
+	ctx := context.Background()
+	for _, bad := range []string{"", "ayse", "ayse@", "@example.com", "a b@example.com", "a@b", "a@@b.com"} {
+		if err := svc.SignUp(ctx, bad, "A", goodPassword); !errors.Is(err, ErrInvalidEmail) {
+			t.Errorf("SignUp(%q) = %v", bad, err)
+		}
+	}
+}
+
+func must(t *testing.T, err error) {
+	t.Helper()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestChangingAPasswordNeedsTheOldOne(t *testing.T) {
+	// A session is what an attacker gets from a borrowed laptop or a stolen
+	// cookie. The password is what they do not have, and without this check the
+	// first thing they would do is take the account.
+	svc, st := newStoreAndUser(t)
+	ctx := context.Background()
+	must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
+	u, _ := st.UserByID(ctx, "usr-1")
+
+	if err := svc.ChangePassword(ctx, st, u, "not the password", "a brand new passphrase"); !errors.Is(err, ErrBadCredentials) {
+		t.Errorf("a wrong current password was accepted: %v", err)
+	}
+	// And the old one still works.
+	if _, _, err := svc.SignIn(ctx, "ayse@example.com", goodPassword); err != nil {
+		t.Errorf("the password changed anyway: %v", err)
+	}
+}
+
+func TestChangingAPasswordSignsOutEverywhere(t *testing.T) {
+	// Leaving other sessions open would mean a password change does nothing
+	// about the reason someone is changing it.
+	svc, st := newStoreAndUser(t)
+	ctx := context.Background()
+	must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
+
+	laptop, _, _ := svc.SignIn(ctx, "ayse@example.com", goodPassword)
+	phone, _, _ := svc.SignIn(ctx, "ayse@example.com", goodPassword)
+
+	u, _ := st.UserByID(ctx, "usr-1")
+	must(t, svc.ChangePassword(ctx, st, u, goodPassword, "a brand new passphrase"))
+
+	for name, token := range map[string]string{"laptop": laptop, "phone": phone} {
+		if _, _, err := svc.Resolve(ctx, token); !errors.Is(err, store.ErrNoSession) {
+			t.Errorf("the %s session survived the change: %v", name, err)
+		}
+	}
+	// The new password works and the old one does not.
+	if _, _, err := svc.SignIn(ctx, "ayse@example.com", "a brand new passphrase"); err != nil {
+		t.Errorf("the new password does not work: %v", err)
+	}
+	if _, _, err := svc.SignIn(ctx, "ayse@example.com", goodPassword); !errors.Is(err, ErrBadCredentials) {
+		t.Errorf("the old password still works: %v", err)
+	}
+}
+
+func TestANewPasswordMustActuallyBeNew(t *testing.T) {
+	svc, st := newStoreAndUser(t)
+	ctx := context.Background()
+	must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
+	u, _ := st.UserByID(ctx, "usr-1")
+
+	if err := svc.ChangePassword(ctx, st, u, goodPassword, goodPassword); !errors.Is(err, ErrSamePassword) {
+		t.Errorf("changing a password to itself: %v", err)
+	}
+	if err := svc.ChangePassword(ctx, st, u, goodPassword, "short"); !errors.Is(err, ErrWeakPassword) {
+		t.Errorf("a short new password was accepted: %v", err)
+	}
+}
+
+func TestGuessingTheCurrentPasswordIsBounded(t *testing.T) {
+	// Someone with a stolen session should not get unlimited attempts at the
+	// password that would let them keep it.
+	st := newStore(t)
+	ctx := context.Background()
+	hash, _ := HashPassword(goodPassword)
+	must(t, st.CreateUser(ctx, store.User{
+		ID: "usr-1", Email: "a@example.com", Name: "A",
+		PasswordHash: hash, State: store.UserPending, CreatedAt: time.Now(),
+	}))
+	must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
+
+	cfg := DefaultConfig()
+	cfg.MaxAttempts = 3
+	svc := New(st, cfg, slog.New(slog.DiscardHandler))
+	u, _ := st.UserByID(ctx, "usr-1")
+
+	for range 3 {
+		_ = svc.ChangePassword(ctx, st, u, "wrong", "a brand new passphrase")
+	}
+	if err := svc.ChangePassword(ctx, st, u, "wrong", "a brand new passphrase"); !errors.Is(err, ErrTooManyTries) {
+		t.Errorf("guessing was unbounded: %v", err)
+	}
+}

internal/account/password.go

@@ -0,0 +1,112 @@
+// Package account is the human half of authentication: people sign up, an
+// operator approves them, and they issue their own API keys from a profile page
+// (plan §7.9).
+//
+// This is deliberately a different mechanism from the one in internal/auth.
+// That one authenticates machines holding 256-bit random keys; this one
+// authenticates people holding passwords they chose. The two have opposite
+// requirements — a fast hash is correct there and wrong here — and keeping them
+// apart is what stops one set of assumptions leaking into the other.
+package account
+
+import (
+	"crypto/rand"
+	"crypto/subtle"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"strings"
+
+	"golang.org/x/crypto/argon2"
+)
+
+// Argon2 parameters. OWASP's current floor for argon2id, which costs roughly
+// 50ms per attempt on ordinary hardware — slow enough to make an offline
+// dictionary attack expensive, fast enough that a login does not feel broken.
+//
+// They are stored with each hash, so raising them later leaves existing
+// passwords working until their owners next sign in.
+const (
+	argonTime    = 3
+	argonMemory  = 64 * 1024 // KiB
+	argonThreads = 4
+	argonKeyLen  = 32
+	saltLen      = 16
+)
+
+// MinPasswordLen is the only password rule.
+//
+// Length is the one requirement that reliably buys entropy. Composition rules —
+// a digit, a symbol, a capital — push people towards Password1! and produce
+// less entropy than they cost, so there are none.
+const MinPasswordLen = 12
+
+// ErrWeakPassword says a password is too short to accept.
+var ErrWeakPassword = fmt.Errorf("a password must be at least %d characters", MinPasswordLen)
+
+// ErrBadHash says a stored hash could not be parsed.
+var ErrBadHash = errors.New("unreadable password hash")
+
+// HashPassword derives a storable hash.
+func HashPassword(password string) (string, error) {
+	if len([]rune(password)) < MinPasswordLen {
+		return "", ErrWeakPassword
+	}
+	salt := make([]byte, saltLen)
+	if _, err := rand.Read(salt); err != nil {
+		return "", err
+	}
+	sum := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen)
+
+	b64 := base64.RawStdEncoding.EncodeToString
+	return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s",
+		argon2.Version, argonMemory, argonTime, argonThreads, b64(salt), b64(sum)), nil
+}
+
+// VerifyPassword reports whether a password matches a stored hash.
+//
+// The comparison is constant time. The parameters come from the hash rather
+// than from the constants above, so a hash written under older settings still
+// verifies.
+func VerifyPassword(password, encoded string) (bool, error) {
+	parts := strings.Split(encoded, "$")
+	if len(parts) != 6 || parts[1] != "argon2id" {
+		return false, ErrBadHash
+	}
+
+	var version int
+	if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil || version != argon2.Version {
+		return false, ErrBadHash
+	}
+	var memory, time uint32
+	var threads uint8
+	if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &memory, &time, &threads); err != nil {
+		return false, ErrBadHash
+	}
+
+	salt, err := base64.RawStdEncoding.DecodeString(parts[4])
+	if err != nil {
+		return false, ErrBadHash
+	}
+	want, err := base64.RawStdEncoding.DecodeString(parts[5])
+	if err != nil {
+		return false, ErrBadHash
+	}
+
+	got := argon2.IDKey([]byte(password), salt, time, memory, threads, uint32(len(want)))
+	return subtle.ConstantTimeCompare(got, want) == 1, nil
+}
+
+// dummyHash is verified against when no account matches, so a login attempt for
+// an address that does not exist costs the same time as one that does.
+//
+// Without it, a stopwatch enumerates every account on the system.
+var dummyHash string
+
+func init() {
+	h, err := HashPassword(strings.Repeat("x", MinPasswordLen))
+	if err != nil {
+		panic("account: cannot build the dummy hash: " + err.Error())
+	}
+	dummyHash = h
+}

internal/account/service.go

@@ -0,0 +1,330 @@
+package account
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/subtle"
+	"encoding/base32"
+	"errors"
+	"log/slog"
+	"strings"
+	"sync"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+// What a sign-in or sign-up can fail with.
+var (
+	ErrBadCredentials = errors.New("that email and password do not match an account")
+	ErrNotApproved    = errors.New("this account is waiting for approval")
+	ErrSuspended      = errors.New("this account has been suspended")
+	ErrTooManyTries   = errors.New("too many attempts; try again shortly")
+	ErrInvalidEmail   = errors.New("that does not look like an email address")
+)
+
+// Store is what the service needs from persistence.
+type Store interface {
+	CreateUser(ctx context.Context, u store.User) error
+	UserByEmail(ctx context.Context, email string) (store.User, error)
+	UserByID(ctx context.Context, id string) (store.User, error)
+	TouchLogin(ctx context.Context, id string, at time.Time) error
+
+	CreateSession(ctx context.Context, userID, token, csrf string, ttl time.Duration) error
+	SessionByToken(ctx context.Context, token string) (store.Session, store.User, error)
+	DeleteSession(ctx context.Context, token string) error
+	TouchSession(ctx context.Context, token string, at time.Time) error
+}
+
+// Config tunes the service.
+type Config struct {
+	// SignupOpen decides whether anyone may create an account. Off, the form is
+	// not served at all; accounts are made with `llmgw users create`.
+	SignupOpen bool
+	// SessionTTL is how long a sign-in lasts.
+	SessionTTL time.Duration
+	// MaxAttempts and AttemptWindow bound password guessing per address.
+	MaxAttempts   int
+	AttemptWindow time.Duration
+}
+
+// DefaultConfig returns sensible values.
+func DefaultConfig() Config {
+	return Config{
+		SignupOpen:    true,
+		SessionTTL:    store.SessionTTL,
+		MaxAttempts:   10,
+		AttemptWindow: 15 * time.Minute,
+	}
+}
+
+func (c *Config) normalize() {
+	d := DefaultConfig()
+	if c.SessionTTL <= 0 {
+		c.SessionTTL = d.SessionTTL
+	}
+	if c.MaxAttempts <= 0 {
+		c.MaxAttempts = d.MaxAttempts
+	}
+	if c.AttemptWindow <= 0 {
+		c.AttemptWindow = d.AttemptWindow
+	}
+}
+
+// Service is signup, sign-in and sessions.
+type Service struct {
+	store Store
+	cfg   Config
+	log   *slog.Logger
+	now   func() time.Time
+
+	mu       sync.Mutex
+	attempts map[string]*attemptWindow
+}
+
+type attemptWindow struct {
+	count int
+	since time.Time
+}
+
+// New builds the service.
+func New(s Store, cfg Config, log *slog.Logger) *Service {
+	cfg.normalize()
+	if log == nil {
+		log = slog.Default()
+	}
+	return &Service{store: s, cfg: cfg, log: log, now: time.Now, attempts: map[string]*attemptWindow{}}
+}
+
+// SignupOpen reports whether the form should be served.
+func (s *Service) SignupOpen() bool { return s.cfg.SignupOpen }
+
+// SignUp creates a pending account.
+//
+// It returns nil whether or not the address was already taken. A signup form
+// that says "this address already has an account" is an account enumerator that
+// anyone can run, and the person signing up learns nothing they need: either
+// way, what happens next is that they wait for approval.
+func (s *Service) SignUp(ctx context.Context, email, name, password string) error {
+	email = store.NormalizeEmail(email)
+	if !validEmail(email) {
+		return ErrInvalidEmail
+	}
+	hash, err := HashPassword(password)
+	if err != nil {
+		return err
+	}
+	if name = strings.TrimSpace(name); name == "" {
+		name = email
+	}
+
+	err = s.store.CreateUser(ctx, store.User{
+		ID: randomID("usr"), Email: email, Name: name,
+		PasswordHash: hash, State: store.UserPending, CreatedAt: s.now(),
+	})
+	if errors.Is(err, store.ErrEmailTaken) {
+		// Logged so an operator can see it; not told to the caller.
+		s.log.Info("signup for an address that already has an account")
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	s.log.Info("a new account is waiting for approval", "email", email)
+	return nil
+}
+
+// SignIn checks a password and opens a session, returning the session token and
+// the CSRF token that must accompany the forms in it.
+func (s *Service) SignIn(ctx context.Context, email, password string) (token, csrf string, err error) {
+	email = store.NormalizeEmail(email)
+	if !s.allowAttempt(email) {
+		return "", "", ErrTooManyTries
+	}
+
+	u, err := s.store.UserByEmail(ctx, email)
+	if errors.Is(err, store.ErrNotFound) {
+		// Verify anyway, against a hash that cannot match. Returning early here
+		// would let a stopwatch enumerate every account on the system.
+		_, _ = VerifyPassword(password, dummyHash)
+		return "", "", ErrBadCredentials
+	}
+	if err != nil {
+		return "", "", err
+	}
+
+	ok, err := VerifyPassword(password, u.PasswordHash)
+	if err != nil || !ok {
+		return "", "", ErrBadCredentials
+	}
+	// The password was right, so this was not an attack.
+	s.clearAttempts(email)
+
+	// State is checked after the password, on purpose. Checking first would
+	// tell anyone who guesses an address whether it has a pending account.
+	switch u.State {
+	case store.UserPending:
+		return "", "", ErrNotApproved
+	case store.UserSuspended:
+		return "", "", ErrSuspended
+	}
+
+	if token, err = store.NewSessionToken(); err != nil {
+		return "", "", err
+	}
+	if csrf, err = store.NewSessionToken(); err != nil {
+		return "", "", err
+	}
+	if err := s.store.CreateSession(ctx, u.ID, token, csrf, s.cfg.SessionTTL); err != nil {
+		return "", "", err
+	}
+	if err := s.store.TouchLogin(ctx, u.ID, s.now()); err != nil {
+		s.log.Warn("could not record a sign-in", "err", err)
+	}
+	return token, csrf, nil
+}
+
+// Resolve turns a session cookie into the person holding it.
+func (s *Service) Resolve(ctx context.Context, token string) (store.Session, store.User, error) {
+	if token == "" {
+		return store.Session{}, store.User{}, store.ErrNoSession
+	}
+	sess, u, err := s.store.SessionByToken(ctx, token)
+	if err != nil {
+		return store.Session{}, store.User{}, err
+	}
+	if err := s.store.TouchSession(ctx, token, s.now()); err != nil {
+		s.log.Debug("could not touch a session", "err", err)
+	}
+	return sess, u, nil
+}
+
+// SignOut closes one session.
+func (s *Service) SignOut(ctx context.Context, token string) error {
+	if token == "" {
+		return nil
+	}
+	return s.store.DeleteSession(ctx, token)
+}
+
+// CheckCSRF compares a submitted token against the session's.
+//
+// A cookie authenticates the browser, not the page that asked. Without this, any
+// site the person visits can post a form to ours and have the browser attach the
+// session automatically.
+func (s *Service) CheckCSRF(sess store.Session, submitted string) bool {
+	if submitted == "" || len(sess.CSRFHash) == 0 {
+		return false
+	}
+	return subtle.ConstantTimeCompare(store.HashKey(submitted), sess.CSRFHash) == 1
+}
+
+// allowAttempt bounds password guessing per address.
+//
+// Per address rather than per IP: an attacker has many addresses to come from
+// and only one to guess against, and limiting by IP would lock out everyone
+// behind one office NAT the moment somebody fat-fingers their password.
+func (s *Service) allowAttempt(email string) bool {
+	now := s.now()
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	w, ok := s.attempts[email]
+	if !ok || now.Sub(w.since) > s.cfg.AttemptWindow {
+		s.attempts[email] = &attemptWindow{count: 1, since: now}
+		return true
+	}
+	w.count++
+	return w.count <= s.cfg.MaxAttempts
+}
+
+func (s *Service) clearAttempts(email string) {
+	s.mu.Lock()
+	delete(s.attempts, email)
+	s.mu.Unlock()
+}
+
+// validEmail is deliberately crude. A real check is delivery, which this does
+// not do; this only rejects what is obviously not an address so the database
+// does not fill with them.
+func validEmail(s string) bool {
+	at := strings.IndexByte(s, '@')
+	if at <= 0 || at != strings.LastIndexByte(s, '@') || at == len(s)-1 {
+		return false
+	}
+	if strings.ContainsAny(s, " \t\r\n<>\"") {
+		return false
+	}
+	return strings.Contains(s[at+1:], ".")
+}
+
+func randomID(prefix string) string {
+	var buf [10]byte
+	_, _ = rand.Read(buf[:])
+	return prefix + "-" + strings.ToLower(
+		base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(buf[:]))
+}
+
+// ErrSamePassword says the new password is the old one.
+var ErrSamePassword = errors.New("that is the password you already have")
+
+// PasswordChanger is the extra storage a password change needs.
+type PasswordChanger interface {
+	SetPassword(ctx context.Context, userID, hash string) error
+	DeleteUserSessions(ctx context.Context, userID string) error
+}
+
+// ChangePassword replaces a person's password and signs them out everywhere.
+//
+// The current password is required even though the caller already holds a
+// session. A session is what an attacker gets from a borrowed laptop or a stolen
+// cookie; the password is what they do not have, and without this check the
+// first thing they would do is take the account.
+//
+// Every session is closed, this browser's included. Leaving the others open
+// would mean a password change does nothing about the reason someone is
+// changing it. The caller re-issues a session for the browser in front of them.
+func (s *Service) ChangePassword(ctx context.Context, pw PasswordChanger,
+	u store.User, current, next string,
+) error {
+	if !s.allowAttempt("pw:" + u.ID) {
+		return ErrTooManyTries
+	}
+	ok, err := VerifyPassword(current, u.PasswordHash)
+	if err != nil || !ok {
+		return ErrBadCredentials
+	}
+	s.clearAttempts("pw:" + u.ID)
+
+	if same, _ := VerifyPassword(next, u.PasswordHash); same {
+		return ErrSamePassword
+	}
+	hash, err := HashPassword(next)
+	if err != nil {
+		return err
+	}
+	if err := pw.SetPassword(ctx, u.ID, hash); err != nil {
+		return err
+	}
+	if err := pw.DeleteUserSessions(ctx, u.ID); err != nil {
+		return err
+	}
+	s.log.Info("a password was changed", "user", u.ID)
+	return nil
+}
+
+// OpenSession starts a session for a person who has just proved who they are by
+// some other means — immediately after changing their password, which closed
+// the one they were using.
+func (s *Service) OpenSession(ctx context.Context, userID string) (token, csrf string, err error) {
+	if token, err = store.NewSessionToken(); err != nil {
+		return "", "", err
+	}
+	if csrf, err = store.NewSessionToken(); err != nil {
+		return "", "", err
+	}
+	if err := s.store.CreateSession(ctx, userID, token, csrf, s.cfg.SessionTTL); err != nil {
+		return "", "", err
+	}
+	return token, csrf, nil
+}

internal/accounting/accounting.go

@@ -0,0 +1,371 @@
+package accounting
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+// Refusals a key can hit. They are distinguished so the client is told which
+// limit it hit, and so retrying is only suggested where retrying helps.
+var (
+	ErrRateLimited  = errors.New("request rate limit exceeded")
+	ErrTokenLimited = errors.New("token rate limit exceeded")
+	ErrBudgetSpent  = errors.New("budget exhausted")
+)
+
+// LimitError carries which limit was hit and, where it makes sense, when to
+// come back.
+type LimitError struct {
+	Err error
+	// RetryAfter is zero when waiting does not help, which is the case for a
+	// spent budget: it will not refill until the period rolls over, and telling
+	// a client to retry in a second would be a lie.
+	RetryAfter time.Duration
+}
+
+func (e *LimitError) Error() string { return e.Err.Error() }
+func (e *LimitError) Unwrap() error { return e.Err }
+
+// Accountant enforces limits and records what was used.
+//
+// One object because the three halves are not independent: a budget check needs
+// the price, the price needs the model, and the record closes the loop by
+// feeding the counters that the next check reads.
+type Accountant struct {
+	pricing  *Pricing
+	budgets  *Budgets
+	requests *Limiter
+	tokens   *Limiter
+	recorder *Recorder
+	log      *slog.Logger
+}
+
+// New builds an accountant.
+func New(pricing *Pricing, recorder *Recorder, log *slog.Logger) *Accountant {
+	if log == nil {
+		log = slog.Default()
+	}
+	return &Accountant{
+		pricing:  pricing,
+		budgets:  NewBudgets(),
+		requests: NewLimiter(),
+		tokens:   NewLimiter(),
+		recorder: recorder,
+		log:      log,
+	}
+}
+
+// RestoreBudgets seeds the in-memory counters from the store.
+//
+// Called at startup. A restart must not reset a budget, which it would if the
+// counters only ever lived in memory.
+func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period store.Period) error {
+	start := PeriodStart(period, time.Now())
+	if start.IsZero() {
+		return nil
+	}
+	rows, err := s.SpendSinceByScope(ctx, start)
+	if err != nil {
+		return fmt.Errorf("restoring budgets: %w", err)
+	}
+	// Every scope a key sits under, not just the key. A budget owned by a team
+	// is one counter fed by every key beneath it, and restoring only per-key
+	// totals would bring the control plane back up believing the team had spent
+	// nothing — which is a budget that resets on every restart, for exactly the
+	// scopes where it matters most.
+	spend := make(map[string]int64, len(rows)*4)
+	for _, r := range rows {
+		for _, o := range []store.LimitOwner{
+			{Scope: store.ScopeKey, ID: r.KeyID},
+			{Scope: store.ScopeProject, ID: r.ProjectID},
+			{Scope: store.ScopeTeam, ID: r.TeamID},
+			{Scope: store.ScopeOrg, ID: r.OrgID},
+		} {
+			if k := o.CounterKey(); k != "" {
+				spend[k] += r.Micros
+			}
+		}
+	}
+	a.budgets.Restore(period, spend)
+	a.log.Info("restored budgets from the store",
+		"keys", len(rows), "counters", len(spend), "since", start)
+	return nil
+}
+
+// Reservation is what Check returns: the estimate it charged, so Record can
+// settle the difference against the real figure.
+// pool is one level's counter and the period its budget runs over.
+type pool struct {
+	Counter string
+	Period  store.Period
+}
+
+type Reservation struct {
+	KeyID string
+	// Budgets and Tokens are every counter the check went through, carried so
+	// the settlement lands on the same ones. Every, not the tightest: an
+	// organization's pool only fills up if each team's spend is added to it
+	// as well as to the team's own.
+	//
+	// Derived again at settlement would mean re-resolving the limits, and a
+	// budget checked against the team and settled against the key is a budget
+	// that never fills up.
+	Budgets         []pool
+	Tokens          []string
+	EstimatedTokens int64
+	Price           Price
+	// Period is the key's budget period, carried from the check so the record
+	// counts against the same window the check read. Deriving it later would
+	// mean guessing, and a monthly budget credited to a daily counter is a
+	// budget that never binds.
+	Period store.Period
+}
+
+// Check decides whether a request may proceed.
+//
+// It runs before scheduling and touches no database: every counter it reads is
+// in memory. The token limit is checked against an estimate, because the real
+// figure only exists once the request has run; Record settles the difference.
+func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64) (Reservation, error) {
+	price := a.pricing.For(model)
+	lim := p.Limits
+	// Every level that set a limit, each against its own pool. A quota written
+	// on a team is one pool the team shares, and the organization above it is a
+	// second pool that the team's spending also fills — both have to hold, or
+	// an organization given 1000 with four teams given 500 spends 2000.
+	chain := levels(lim, p.KeyID)
+
+	res := Reservation{
+		KeyID: p.KeyID, EstimatedTokens: estimatedTokens,
+		Price: price, Period: lim.BudgetPeriod,
+	}
+	for _, l := range chain {
+		if l.Budget != nil {
+			res.Budgets = append(res.Budgets, pool{l.Counter, l.Period})
+		}
+		if l.TPM != nil {
+			res.Tokens = append(res.Tokens, l.Counter)
+		}
+	}
+	if p.KeyID == "" {
+		return res, nil
+	}
+
+	// Budget first. It is the limit whose refusal cannot be retried away, so
+	// telling the client about it first is more useful than reporting a rate
+	// limit it would hit again in a second.
+	//
+	// A model that costs nothing is exempt, which is what lets a key with no
+	// budget left still be served a cheap fallback (plan §7.7).
+	// Budgets first, and all of them. Exhausted only reads, so the order costs
+	// nothing and a refusal here charges no counter.
+	if !price.Zero() {
+		for _, l := range chain {
+			if l.Budget != nil && a.budgets.Exhausted(l.Counter, l.Period, *l.Budget) {
+				return res, &LimitError{Err: ErrBudgetSpent}
+			}
+		}
+	}
+
+	// Rate limits charge as they check — deliberately, so two concurrent
+	// requests cannot both see room. With several levels that means a refusal
+	// at one has already taken from the levels before it, so those are handed
+	// back. Without the refund a team at its ceiling would quietly eat its
+	// organization's rate limit with requests that were never served.
+	if err := a.allowAll(chain, func(l level) (string, int, int64, bool) {
+		return l.Counter, valInt(l.RPM), 1, l.RPM != nil
+	}, a.requests); err != nil {
+		return res, &LimitError{Err: ErrRateLimited, RetryAfter: time.Second}
+	}
+	if err := a.allowAll(chain, func(l level) (string, int, int64, bool) {
+		return l.Counter, valInt(l.TPM), estimatedTokens, l.TPM != nil
+	}, a.tokens); err != nil {
+		// The request was not charged against the request-rate limit in vain:
+		// it was, and that is correct — it did arrive.
+		return res, &LimitError{Err: ErrTokenLimited, RetryAfter: time.Second}
+	}
+	return res, nil
+}
+
+// level is one scope's limits with the counter they are enforced against.
+type level struct {
+	Counter string
+	Period  store.Period
+	RPM     *int
+	TPM     *int
+	Budget  *int64
+}
+
+// levels turns a resolved chain into the pools a request must pass.
+//
+// Falls back to the collapsed values when the chain is empty, which is what a
+// principal assembled by hand looks like — in tests, and anywhere that builds
+// limits without walking the tree.
+func levels(e store.Effective, keyID string) []level {
+	if len(e.Chain) == 0 {
+		var out []level
+		if e.RPM != nil || e.TPM != nil || e.Budget != nil {
+			out = append(out, level{
+				Counter: counterFor(e.BudgetOwner, keyID),
+				Period:  e.BudgetPeriod, Budget: e.Budget,
+			})
+			out[0].RPM, out[0].TPM = e.RPM, e.TPM
+			// One entry, so a hand-built principal with different owners per
+			// limit still counts each where it said to.
+			if r, b := counterFor(e.RPMOwner, keyID), out[0].Counter; r != b && e.RPM != nil {
+				out[0].RPM = nil
+				out = append(out, level{Counter: r, RPM: e.RPM})
+			}
+			if tp, b := counterFor(e.TPMOwner, keyID), out[0].Counter; tp != b && e.TPM != nil {
+				out[0].TPM = nil
+				out = append(out, level{Counter: tp, TPM: e.TPM})
+			}
+		}
+		return out
+	}
+	out := make([]level, 0, len(e.Chain))
+	for _, l := range e.Chain {
+		out = append(out, level{
+			Counter: store.LimitOwner{Scope: l.Scope, ID: l.ScopeID}.CounterKey(),
+			Period:  l.BudgetPeriod, RPM: l.RPM, TPM: l.TPM, Budget: l.Budget,
+		})
+	}
+	return out
+}
+
+// allowAll charges every level that has this limit, refunding on a refusal.
+func (a *Accountant) allowAll(chain []level,
+	pick func(level) (key string, limit int, n int64, has bool), lim *Limiter,
+) error {
+	var charged []struct {
+		key string
+		n   int64
+	}
+	for _, l := range chain {
+		key, limit, n, has := pick(l)
+		if !has {
+			continue
+		}
+		if !lim.Allow(key, limit, n) {
+			for _, c := range charged {
+				lim.Charge(c.key, -c.n)
+			}
+			return errRefused
+		}
+		charged = append(charged, struct {
+			key string
+			n   int64
+		}{key, n})
+	}
+	return nil
+}
+
+var errRefused = errors.New("refused")
+
+func valInt(v *int) int {
+	if v == nil {
+		return 0
+	}
+	return *v
+}
+
+// Record settles a finished request: it reconciles the token estimate, adds to
+// the budget, and queues the usage row.
+func (a *Accountant) Record(res Reservation, u store.Usage) {
+	if res.KeyID == "" {
+		return
+	}
+	actual := u.PromptTokens + u.CompletionTokens
+
+	// The estimate was charged up front; charge the difference now. A caller
+	// whose requests are consistently larger than estimated is still bounded,
+	// one window later than with a perfect estimate.
+	// Every level the check went through, so an organization's pool fills up
+	// from its teams' spending rather than only from requests that named it.
+	for _, c := range res.Tokens {
+		a.tokens.Charge(c, actual-res.EstimatedTokens)
+	}
+
+	u.CostMicros = res.Price.Cost(u.PromptTokens, u.CompletionTokens)
+	for _, b := range res.Budgets {
+		a.budgets.Add(b.Counter, b.Period, u.CostMicros)
+	}
+
+	if a.recorder != nil {
+		a.recorder.Record(u)
+	}
+}
+
+// Status returns what a key has used, for headers and `ctl`.
+func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) {
+	if p.KeyID == "" {
+		return 0, 0, 0
+	}
+	// The pool that will refuse them, which is not always the one with the
+	// smallest number on it. A team given 2000 inside an organization given the
+	// same is bounded by whichever is nearer its ceiling, and that is the
+	// organization as soon as a second team spends anything. A header that
+	// reported the team's would disagree with the refusal that follows it.
+	for _, l := range levels(p.Limits, p.KeyID) {
+		if l.Budget != nil {
+			if v := a.budgets.Spent(l.Counter, l.Period); v > spent {
+				spent = v
+			}
+		}
+		if l.RPM != nil {
+			if v := a.requests.Rate(l.Counter); v > rpm {
+				rpm = v
+			}
+		}
+		if l.TPM != nil {
+			if v := a.tokens.Rate(l.Counter); v > tpm {
+				tpm = v
+			}
+		}
+	}
+	return spent, rpm, tpm
+}
+
+// counterFor is the counter a limit owned by this scope should use.
+//
+// Falls back to the key's own counter when no scope owns the limit, which
+// happens when nothing in the chain set one.
+//
+// Namespaced even then. A bare key id here and a "key:"-prefixed one in the
+// restore path would be two different counters for the same key, so a budget
+// would check against an empty one and survive every restart — which is how
+// this was wrong when it was first written.
+func counterFor(o store.LimitOwner, keyID string) string {
+	if k := o.CounterKey(); k != "" {
+		return k
+	}
+	return keyCounter(keyID)
+}
+
+// keyCounter is a key's own counter, in the same namespace as every other.
+func or(v, fallback string) string {
+	if v == "" {
+		return fallback
+	}
+	return v
+}
+
+func keyCounter(keyID string) string {
+	if keyID == "" {
+		return ""
+	}
+	return store.LimitOwner{Scope: store.ScopeKey, ID: keyID}.CounterKey()
+}
+
+// AnythingPriced reports whether a budget can ever be spent against.
+//
+// On the accountant rather than making callers reach for the price table: what
+// they are asking is whether the budgets they are about to display mean
+// anything, and the accountant is what would enforce them.
+func (a *Accountant) AnythingPriced() bool { return a.pricing.AnythingPriced() }

internal/accounting/accounting_test.go

@@ -0,0 +1,632 @@
+package accounting
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+func intp(v int) *int     { return &v }
+func i64p(v int64) *int64 { return &v }
+
+func principal(id string, tune func(*store.Effective)) auth.Principal {
+	e := store.Effective{KeyID: id, ProjectID: "proj"}
+	if tune != nil {
+		tune(&e)
+	}
+	return auth.Principal{KeyID: id, ProjectID: "proj", Limits: e}
+}
+
+func pricing(t *testing.T) *Pricing {
+	t.Helper()
+	return NewPricing(map[string]Price{
+		"paid": {PromptPer1k: 1000, CompletionPer1k: 3000},
+		"free": {},
+	}, Price{})
+}
+
+func TestCostRoundsUpSoSmallRequestsAreNotFree(t *testing.T) {
+	// Rounding down would make a stream of tiny requests cost nothing, which is
+	// exactly the traffic shape a quota exists to bound.
+	p := Price{PromptPer1k: 1000, CompletionPer1k: 3000}
+	if got := p.Cost(1, 0); got != 1 {
+		t.Errorf("one prompt token cost %d", got)
+	}
+	if got := p.Cost(1000, 1000); got != 4000 {
+		t.Errorf("a thousand of each cost %d, want 4000", got)
+	}
+	if got := p.Cost(0, 0); got != 0 {
+		t.Errorf("an empty request cost %d", got)
+	}
+}
+
+func TestZeroPricedModelsStayAvailableWhenTheBudgetIsSpent(t *testing.T) {
+	// This is what makes budget fallback work: a key with nothing left can
+	// still be served something that costs nothing.
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	p := principal("k1", func(e *store.Effective) {
+		e.Budget = i64p(100)
+		e.BudgetPeriod = store.PeriodMonth
+	})
+
+	res, err := a.Check(p, "paid", 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 0})
+
+	if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) {
+		t.Errorf("a spent budget still served a paid model: %v", err)
+	}
+	if _, err := a.Check(p, "free", 10); err != nil {
+		t.Errorf("a spent budget refused a free model: %v", err)
+	}
+}
+
+func TestBudgetRefusalDoesNotSuggestRetrying(t *testing.T) {
+	// A budget does not refill until the period rolls over, so a Retry-After of
+	// a second would be a lie that turns one client into a retry loop.
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	p := principal("k1", func(e *store.Effective) {
+		e.Budget = i64p(1)
+		e.BudgetPeriod = store.PeriodDay
+	})
+	res, _ := a.Check(p, "paid", 10)
+	a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 5000})
+
+	_, err := a.Check(p, "paid", 10)
+	var le *LimitError
+	if !errors.As(err, &le) {
+		t.Fatalf("expected a limit error, got %v", err)
+	}
+	if le.RetryAfter != 0 {
+		t.Errorf("a spent budget suggested retrying in %v", le.RetryAfter)
+	}
+}
+
+func TestRateLimitsBind(t *testing.T) {
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	p := principal("k1", func(e *store.Effective) { e.RPM = intp(3) })
+
+	for i := range 3 {
+		if _, err := a.Check(p, "free", 1); err != nil {
+			t.Fatalf("request %d was refused: %v", i, err)
+		}
+	}
+	if _, err := a.Check(p, "free", 1); !errors.Is(err, ErrRateLimited) {
+		t.Errorf("the fourth request was allowed: %v", err)
+	}
+}
+
+func TestTokenLimitsUseTheEstimateAndThenSettle(t *testing.T) {
+	// The real token count only exists once the request has run, so the check
+	// has to work on an estimate. A caller that consistently underestimates
+	// must still end up bounded.
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	p := principal("k1", func(e *store.Effective) { e.TPM = intp(1000) })
+
+	// Claims 10 tokens, actually uses 900.
+	res, err := a.Check(p, "free", 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 400, CompletionTokens: 500})
+
+	// The settled figure is what the next check sees, so a second request of
+	// the same shape cannot fit.
+	if _, err := a.Check(p, "free", 200); !errors.Is(err, ErrTokenLimited) {
+		t.Errorf("the underestimate was never settled: %v", err)
+	}
+}
+
+func TestNoLimitIsNotALimitOfZero(t *testing.T) {
+	// The distinction the store keeps with a nil pointer. Getting it wrong here
+	// would silently forbid every request for every key with no limits set.
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	p := principal("k1", nil)
+
+	for range 50 {
+		if _, err := a.Check(p, "paid", 10_000); err != nil {
+			t.Fatalf("a key with no limits was refused: %v", err)
+		}
+	}
+}
+
+func TestAnUnauthenticatedRequestIsNotAccounted(t *testing.T) {
+	// With auth off there is no key to charge, and Check must not refuse.
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	res, err := a.Check(auth.Principal{}, "paid", 1000)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.Record(res, store.Usage{PromptTokens: 1000})
+	if spent, _, _ := a.Status(auth.Principal{}); spent != 0 {
+		t.Errorf("an anonymous request was charged %d", spent)
+	}
+}
+
+func TestBudgetSurvivesARestart(t *testing.T) {
+	// The counters live in memory; the store is what makes them survive.
+	ctx := context.Background()
+	path := filepath.Join(t.TempDir(), "acct.db")
+	s, err := store.OpenSQLite(ctx, path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+
+	must(t, s.PutOrg(ctx, store.Org{ID: "o", Name: "o"}))
+	must(t, s.PutTeam(ctx, store.Team{ID: "t", OrgID: "o", Name: "t"}))
+	must(t, s.PutProject(ctx, store.Project{ID: "proj", TeamID: "t", Name: "p"}))
+	_, k, _ := store.NewKey("k1", "n", "proj")
+	k.CreatedAt = time.Now()
+	must(t, s.CreateKey(ctx, k))
+
+	must(t, s.WriteUsage(ctx, []store.Usage{
+		{At: time.Now(), KeyID: "k1", ProjectID: "proj", Model: "paid",
+			ServedModel: "paid", CostMicros: 5000, Status: 200},
+	}))
+
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	must(t, a.RestoreBudgets(ctx, s, store.PeriodMonth))
+
+	p := principal("k1", func(e *store.Effective) {
+		e.Budget = i64p(5000)
+		e.BudgetPeriod = store.PeriodMonth
+	})
+	if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) {
+		t.Errorf("the budget was reset by the restart: %v", err)
+	}
+}
+
+func TestBudgetsResetAtThePeriodBoundary(t *testing.T) {
+	b := NewBudgets()
+	now := time.Date(2026, 1, 31, 23, 59, 0, 0, time.UTC)
+	b.now = func() time.Time { return now }
+
+	b.Add("k1", store.PeriodMonth, 900)
+	if got := b.Spent("k1", store.PeriodMonth); got != 900 {
+		t.Fatalf("spent %d", got)
+	}
+
+	now = now.Add(2 * time.Minute) // into February
+	if got := b.Spent("k1", store.PeriodMonth); got != 0 {
+		t.Errorf("the counter carried %d across the month boundary", got)
+	}
+}
+
+func TestDailyAndMonthlyBoundariesDiffer(t *testing.T) {
+	b := NewBudgets()
+	now := time.Date(2026, 3, 15, 12, 0, 0, 0, time.UTC)
+	b.now = func() time.Time { return now }
+
+	b.Add("daily", store.PeriodDay, 100)
+	b.Add("monthly", store.PeriodMonth, 100)
+
+	now = now.Add(24 * time.Hour)
+	if got := b.Spent("daily", store.PeriodDay); got != 0 {
+		t.Errorf("the daily counter carried %d across a day", got)
+	}
+	if got := b.Spent("monthly", store.PeriodMonth); got != 100 {
+		t.Errorf("the monthly counter lost %d within a month", 100-got)
+	}
+}
+
+func TestTheWindowDoesNotAllowTwiceTheLimit(t *testing.T) {
+	// The reason for a sliding window. With a fixed one, a caller sends its
+	// whole allowance at the end of a window and again at the start of the
+	// next, so the observed rate is double the limit exactly when the system is
+	// least able to take it.
+	l := NewLimiter()
+	base := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
+	now := base.Add(59 * time.Second)
+	l.now = func() time.Time { return now }
+
+	const limit = 60
+	sent := 0
+	for range limit {
+		if l.Allow("k", limit, 1) {
+			sent++
+		}
+	}
+	if sent != limit {
+		t.Fatalf("the first burst only managed %d", sent)
+	}
+
+	// One second later, in the next fixed window.
+	now = base.Add(61 * time.Second)
+	second := 0
+	for range limit {
+		if l.Allow("k", limit, 1) {
+			second++
+		}
+	}
+	// Some allowance returns as the old window slides out of view, but nowhere
+	// near the whole of it.
+	if second > limit/2 {
+		t.Errorf("a second burst of %d passed immediately after %d; the window is not sliding",
+			second, sent)
+	}
+}
+
+func TestLimiterIsSafeUnderConcurrency(t *testing.T) {
+	// Checking and charging have to be one operation, or two concurrent
+	// requests both see room and both take it.
+	l := NewLimiter()
+	const limit = 100
+
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	allowed := 0
+	for range 500 {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if l.Allow("k", limit, 1) {
+				mu.Lock()
+				allowed++
+				mu.Unlock()
+			}
+		}()
+	}
+	wg.Wait()
+	if allowed != limit {
+		t.Errorf("%d requests passed a limit of %d", allowed, limit)
+	}
+}
+
+func must(t *testing.T, err error) {
+	t.Helper()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// teamMember is a key whose budget and rate limit were set on its team, which
+// is the arrangement an enterprise actually asks for: a group gets a quota and
+// the people in it share it.
+func teamMember(keyID, teamID string, tune func(*store.Effective)) auth.Principal {
+	e := store.Effective{KeyID: keyID, ProjectID: "proj", TeamID: teamID}
+	owner := store.LimitOwner{Scope: store.ScopeTeam, ID: teamID}
+	e.BudgetOwner, e.RPMOwner, e.TPMOwner = owner, owner, owner
+	if tune != nil {
+		tune(&e)
+	}
+	return auth.Principal{KeyID: keyID, ProjectID: "proj", Limits: e}
+}
+
+func TestATeamBudgetIsOnePoolRatherThanOnePerMember(t *testing.T) {
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	budget := func(e *store.Effective) {
+		e.Budget = i64p(3000)
+		e.BudgetPeriod = store.PeriodMonth
+	}
+	first := teamMember("k1", "t1", budget)
+	second := teamMember("k2", "t1", budget)
+
+	// One member spends the lot.
+	res, err := a.Check(first, "paid", 1000)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 1000})
+
+	// The second must now be refused. Counted per key it would have its own
+	// untouched 3000 — a team budget of 3000 with ten keys would permit thirty
+	// thousand, which is the bug this change exists for.
+	if _, err := a.Check(second, "paid", 10); !errors.Is(err, ErrBudgetSpent) {
+		t.Errorf("the team's budget should be shared; the second member got %v", err)
+	}
+}
+
+func TestATeamRateLimitIsSharedToo(t *testing.T) {
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	rpm := func(e *store.Effective) { e.RPM = intp(1) }
+
+	if _, err := a.Check(teamMember("k1", "t1", rpm), "free", 1); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := a.Check(teamMember("k2", "t1", rpm), "free", 1); !errors.Is(err, ErrRateLimited) {
+		t.Error("a rate limit set on the team should be shared by its keys")
+	}
+}
+
+func TestTwoTeamsDoNotShareACounter(t *testing.T) {
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	rpm := func(e *store.Effective) { e.RPM = intp(1) }
+
+	if _, err := a.Check(teamMember("k1", "t1", rpm), "free", 1); err != nil {
+		t.Fatal(err)
+	}
+	// Pooling is the point; pooling the wrong things is the danger. One team
+	// exhausting another's quota would be worse than the bug being fixed.
+	if _, err := a.Check(teamMember("k2", "t2", rpm), "free", 1); err != nil {
+		t.Errorf("a second team should have its own counter, got %v", err)
+	}
+}
+
+func TestAKeyLimitStaysPerKey(t *testing.T) {
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	perKey := func(id string) auth.Principal {
+		return principal(id, func(e *store.Effective) {
+			e.RPM = intp(1)
+			e.RPMOwner = store.LimitOwner{Scope: store.ScopeKey, ID: id}
+		})
+	}
+
+	if _, err := a.Check(perKey("k1"), "free", 1); err != nil {
+		t.Fatal(err)
+	}
+	// The tree still has to be able to say "each of you may do this much".
+	// Sharing a group pool must not become the only thing expressible.
+	if _, err := a.Check(perKey("k2"), "free", 1); err != nil {
+		t.Errorf("a limit owned by the key should not be shared, got %v", err)
+	}
+}
+
+func TestTheDifferentLimitsCanBeOwnedByDifferentScopes(t *testing.T) {
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	// A team budget with a per-key rate limit is the ordinary arrangement: the
+	// group's money, the individual's throughput.
+	mk := func(id string) auth.Principal {
+		e := store.Effective{KeyID: id, TeamID: "t1"}
+		e.Budget, e.BudgetPeriod = i64p(3000), store.PeriodMonth
+		e.BudgetOwner = store.LimitOwner{Scope: store.ScopeTeam, ID: "t1"}
+		e.RPM = intp(1)
+		e.RPMOwner = store.LimitOwner{Scope: store.ScopeKey, ID: id}
+		return auth.Principal{KeyID: id, Limits: e}
+	}
+
+	res, err := a.Check(mk("k1"), "paid", 1000)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 1000})
+
+	// k2's rate limit is its own and untouched; the budget is the team's and
+	// gone. The budget is checked first, so that is what comes back.
+	if _, err := a.Check(mk("k2"), "paid", 10); !errors.Is(err, ErrBudgetSpent) {
+		t.Errorf("expected the shared budget to bite, got %v", err)
+	}
+}
+
+func TestStatusReportsTheCounterThatWillRefuseYou(t *testing.T) {
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	budget := func(e *store.Effective) {
+		e.Budget = i64p(9000)
+		e.BudgetPeriod = store.PeriodMonth
+	}
+	res, err := a.Check(teamMember("k1", "t1", budget), "paid", 1000)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 1000})
+
+	// A member who has spent nothing themselves still sees the team's spend,
+	// because that is the number that will stop them. A header disagreeing with
+	// the refusal it precedes is worse than no header.
+	spent, _, _ := a.Status(teamMember("k2", "t1", budget))
+	if spent == 0 {
+		t.Error("a member should see the pool's spend, not their own")
+	}
+}
+
+func TestATeamBudgetSurvivesARestart(t *testing.T) {
+	// The per-key restore could not do this: it rebuilt a counter named after
+	// the key, while the check reads one named after the team. The control
+	// plane came back believing the team had spent nothing — a budget that
+	// resets on every restart, for exactly the scope where it matters most.
+	ctx := context.Background()
+	s, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "team.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+
+	must(t, s.PutOrg(ctx, store.Org{ID: "o", Name: "o"}))
+	must(t, s.PutTeam(ctx, store.Team{ID: "t1", OrgID: "o", Name: "t"}))
+	must(t, s.PutProject(ctx, store.Project{ID: "proj", TeamID: "t1", Name: "p"}))
+	for _, id := range []string{"k1", "k2"} {
+		_, k, _ := store.NewKey(id, "n", "proj")
+		k.CreatedAt = time.Now()
+		must(t, s.CreateKey(ctx, k))
+	}
+
+	// Two members of one team, 2500 each: neither alone exceeds a 4000 budget,
+	// together they do.
+	must(t, s.WriteUsage(ctx, []store.Usage{
+		{At: time.Now(), KeyID: "k1", ProjectID: "proj", Model: "paid",
+			ServedModel: "paid", CostMicros: 2500, Status: 200},
+		{At: time.Now(), KeyID: "k2", ProjectID: "proj", Model: "paid",
+			ServedModel: "paid", CostMicros: 2500, Status: 200},
+	}))
+
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	must(t, a.RestoreBudgets(ctx, s, store.PeriodMonth))
+
+	p := teamMember("k1", "t1", func(e *store.Effective) {
+		e.Budget = i64p(4000)
+		e.BudgetPeriod = store.PeriodMonth
+	})
+	if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) {
+		t.Errorf("the team's spend should have been restored and pooled: %v", err)
+	}
+}
+
+// inTree is a key under org/team/project, with whatever each level sets.
+func inTree(keyID string, chain ...store.Limits) auth.Principal {
+	e := store.Effective{KeyID: keyID, Chain: chain}
+	// The collapsed values still exist for headers and the profile page; they
+	// are not what enforcement reads any more.
+	for _, l := range chain {
+		if l.Budget != nil && (e.Budget == nil || *l.Budget < *e.Budget) {
+			e.Budget, e.BudgetPeriod = l.Budget, l.BudgetPeriod
+			e.BudgetOwner = store.LimitOwner{Scope: l.Scope, ID: l.ScopeID}
+		}
+	}
+	return auth.Principal{KeyID: keyID, Limits: e}
+}
+
+func orgBudget(id string, v int64) store.Limits {
+	return store.Limits{Scope: store.ScopeOrg, ScopeID: id,
+		Budget: i64p(v), BudgetPeriod: store.PeriodMonth}
+}
+
+func teamBudget(id string, v int64) store.Limits {
+	return store.Limits{Scope: store.ScopeTeam, ScopeID: id,
+		Budget: i64p(v), BudgetPeriod: store.PeriodMonth}
+}
+
+func TestTheSumOfTeamsCannotExceedTheOrganization(t *testing.T) {
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+
+	// Each request costs 800. The organization allows 2000 and each of its four
+	// teams allows 2000, so no team comes close to its own ceiling — but the
+	// organization's fills after three. Taking only the most restrictive value,
+	// the organization's number was never looked at and all four went through.
+	//
+	// A budget stops spending once it is reached rather than refusing the
+	// request that would cross it, so the third is served and the fourth is not.
+	spend := func(team string) error {
+		p := inTree(team+"-k", orgBudget("o", 2000), teamBudget(team, 2000))
+		res, err := a.Check(p, "paid", 1000)
+		if err != nil {
+			return err
+		}
+		a.Record(res, store.Usage{KeyID: team + "-k", PromptTokens: 200, CompletionTokens: 200})
+		return nil
+	}
+
+	total := 0
+	for _, team := range []string{"t1", "t2", "t3", "t4"} {
+		if err := spend(team); err == nil {
+			total++
+		}
+	}
+	if total != 3 {
+		t.Errorf("expected the organization to stop the fourth team, %d got through", total)
+	}
+}
+
+func TestATeamStillStopsBeforeItsOrganizationDoes(t *testing.T) {
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	p := func() auth.Principal {
+		return inTree("k1", orgBudget("o", 100_000), teamBudget("t1", 800))
+	}
+
+	res, err := a.Check(p(), "paid", 1000)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 200, CompletionTokens: 200})
+
+	// The narrower pool binds first, which is what it is for. Enforcing every
+	// level must not stop the tighter one working.
+	if _, err := a.Check(p(), "paid", 10); !errors.Is(err, ErrBudgetSpent) {
+		t.Errorf("the team's own budget should still bite, got %v", err)
+	}
+}
+
+func TestSpendingFillsEveryPoolAboveIt(t *testing.T) {
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	p := inTree("k1", orgBudget("o", 5000), teamBudget("t1", 5000))
+
+	res, err := a.Check(p, "paid", 1000)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 200, CompletionTokens: 200})
+
+	// Settled onto both, or the organization's pool would only ever fill from
+	// requests that named it directly — which is none of them.
+	if got := a.budgets.Spent("org:o", store.PeriodMonth); got == 0 {
+		t.Error("the organization's pool did not fill")
+	}
+	if got := a.budgets.Spent("team:t1", store.PeriodMonth); got == 0 {
+		t.Error("the team's pool did not fill")
+	}
+}
+
+func TestARefusalHandsBackWhatTheNarrowerLevelsTook(t *testing.T) {
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	// The team allows plenty; the organization allows one request a minute.
+	chain := []store.Limits{
+		{Scope: store.ScopeOrg, ScopeID: "o", RPM: intp(1)},
+		{Scope: store.ScopeTeam, ScopeID: "t1", RPM: intp(100)},
+	}
+	if _, err := a.Check(inTree("k1", chain...), "free", 1); err != nil {
+		t.Fatal(err)
+	}
+	before := a.requests.Rate("team:t1")
+
+	if _, err := a.Check(inTree("k2", chain...), "free", 1); !errors.Is(err, ErrRateLimited) {
+		t.Fatalf("the organization should have refused the second, got %v", err)
+	}
+
+	// A rate limiter charges as it checks, so a refusal further along has
+	// already taken from the levels before it. Left there, a team at its
+	// organization's ceiling would quietly eat its own rate limit with requests
+	// that were never served.
+	if after := a.requests.Rate("team:t1"); after != before {
+		t.Errorf("the refused request stayed charged to the team: %v -> %v", before, after)
+	}
+}
+
+func TestAChainWithNoLimitsAnywhereAllowsEverything(t *testing.T) {
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	for i := 0; i < 5; i++ {
+		if _, err := a.Check(inTree("k1"), "paid", 1000); err != nil {
+			t.Fatalf("nothing is limited, but request %d was refused: %v", i, err)
+		}
+	}
+}
+
+func TestStatusReportsTheFullestPoolNotTheSmallestNumber(t *testing.T) {
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	chain := func(team string) auth.Principal {
+		return inTree(team+"-k", orgBudget("o", 9000), teamBudget(team, 9000))
+	}
+
+	// Another team spends; this one has spent nothing of its own.
+	res, err := a.Check(chain("t1"), "paid", 1000)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.Record(res, store.Usage{KeyID: "t1-k", PromptTokens: 200, CompletionTokens: 200})
+
+	spent, _, _ := a.Status(chain("t2"))
+	// The organization's pool is the one nearer its ceiling, so it is the one
+	// that will refuse them. Reporting the team's nil would disagree with the
+	// refusal that eventually follows it.
+	if spent == 0 {
+		t.Error("a member should see the fullest pool that bounds them")
+	}
+}
+
+func TestAnEmptyPriceTableMeansNoBudgetCanEverBind(t *testing.T) {
+	// The ordinary case for self-hosted models, and the reason a budget on a
+	// team can read as a limit while binding on nothing.
+	if NewPricing(nil, Price{}).AnythingPriced() {
+		t.Error("nothing priced should say so")
+	}
+	if NewPricing(map[string]Price{"m": {}}, Price{}).AnythingPriced() {
+		t.Error("a model priced at zero is not priced")
+	}
+	if !NewPricing(map[string]Price{"m": {PromptPer1k: 1}}, Price{}).AnythingPriced() {
+		t.Error("one priced model is enough for a budget to move")
+	}
+	if !NewPricing(nil, Price{CompletionPer1k: 1}).AnythingPriced() {
+		t.Error("a fallback price prices everything")
+	}
+	var nilPricing *Pricing
+	if nilPricing.AnythingPriced() {
+		t.Error("no price table at all is not priced")
+	}
+}

internal/accounting/budget.go

@@ -0,0 +1,101 @@
+package accounting
+
+import (
+	"sync"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+// PeriodStart is when the current budget period began.
+//
+// Periods are anchored to UTC rather than to a per-key start date, so every
+// key's day and month boundary is the same instant. A key whose period started
+// when it was created would make "spend this month" mean something different
+// for every key, and make the restore query unanswerable in one pass.
+func PeriodStart(p store.Period, now time.Time) time.Time {
+	u := now.UTC()
+	switch p {
+	case store.PeriodDay:
+		return time.Date(u.Year(), u.Month(), u.Day(), 0, 0, 0, 0, time.UTC)
+	case store.PeriodMonth:
+		return time.Date(u.Year(), u.Month(), 1, 0, 0, 0, 0, time.UTC)
+	default:
+		return time.Time{}
+	}
+}
+
+// Budgets tracks spend per key in memory, so a budget check costs no query.
+//
+// The database catches up through the batch writer. That means the in-memory
+// figure leads the stored one, which is the right direction: a budget that only
+// counted what had already been written would let a burst overspend by however
+// much the writer was behind.
+type Budgets struct {
+	now func() time.Time
+
+	mu    sync.Mutex
+	spend map[string]counter
+}
+
+type counter struct {
+	micros int64
+	since  time.Time
+}
+
+// NewBudgets builds an empty tracker.
+func NewBudgets() *Budgets {
+	return &Budgets{now: time.Now, spend: map[string]counter{}}
+}
+
+// Restore seeds the counters from what the store already recorded.
+//
+// A restart must not reset a budget, which it would if the counter lived only
+// in memory. The caller passes the spend since each period's start.
+func (b *Budgets) Restore(period store.Period, spend map[string]int64) {
+	start := PeriodStart(period, b.now())
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	for key, micros := range spend {
+		b.spend[key] = counter{micros: micros, since: start}
+	}
+}
+
+// Spent returns a key's spend in the current period, resetting it if the period
+// has rolled over.
+func (b *Budgets) Spent(keyID string, period store.Period) int64 {
+	start := PeriodStart(period, b.now())
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.currentLocked(keyID, start).micros
+}
+
+// Add records spend.
+func (b *Budgets) Add(keyID string, period store.Period, micros int64) {
+	if micros == 0 {
+		return
+	}
+	start := PeriodStart(period, b.now())
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	c := b.currentLocked(keyID, start)
+	c.micros += micros
+	b.spend[keyID] = c
+}
+
+// Exhausted reports whether a key has spent its budget.
+func (b *Budgets) Exhausted(keyID string, period store.Period, budget int64) bool {
+	return b.Spent(keyID, period) >= budget
+}
+
+// currentLocked returns the counter, rolled over if the period changed.
+func (b *Budgets) currentLocked(keyID string, start time.Time) counter {
+	c, ok := b.spend[keyID]
+	if !ok || c.since.Before(start) {
+		// Either never seen, or the period rolled over while the process was
+		// running. Both start from zero.
+		c = counter{since: start}
+		b.spend[keyID] = c
+	}
+	return c
+}

internal/accounting/cost.go

@@ -0,0 +1,97 @@
+// Package accounting turns finished requests into spend, enforces what a key
+// is allowed, and gets both into the store without touching the request path
+// (plan §7.9.3, §7.9.4).
+package accounting
+
+import (
+	"strings"
+	"sync"
+)
+
+// Price is what one model costs, in micro-units per thousand tokens.
+//
+// Micro-units rather than a float because spend is accumulated by repeated
+// addition, and repeated addition of floats drifts. What a unit means is the
+// operator's: self-hosted models have no per-token price, so it may be
+// GPU-seconds, a notional rate, or 1000 everywhere — which turns a budget into
+// a token quota.
+type Price struct {
+	PromptPer1k     int64 `koanf:"prompt_per_1k"`
+	CompletionPer1k int64 `koanf:"completion_per_1k"`
+}
+
+// Zero reports whether this model spends nothing.
+//
+// It matters beyond arithmetic: a zero-cost model stays available to a key
+// whose budget is spent, which is what makes the plan's budget fallback work
+// (§7.7).
+func (p Price) Zero() bool { return p.PromptPer1k == 0 && p.CompletionPer1k == 0 }
+
+// Cost computes what a request spent, rounding up.
+//
+// Up rather than to nearest: rounding down would make a stream of tiny requests
+// free, which is exactly the shape of traffic a quota exists to bound.
+func (p Price) Cost(promptTokens, completionTokens int64) int64 {
+	return divUp(promptTokens*p.PromptPer1k, 1000) + divUp(completionTokens*p.CompletionPer1k, 1000)
+}
+
+func divUp(n, d int64) int64 {
+	if n <= 0 {
+		return 0
+	}
+	return (n + d - 1) / d
+}
+
+// Pricing resolves a model name to its price.
+type Pricing struct {
+	mu       sync.RWMutex
+	byModel  map[string]Price
+	fallback Price
+}
+
+// NewPricing builds a price table. Names are matched case-insensitively,
+// because a model name reaches us from whatever the client typed.
+func NewPricing(byModel map[string]Price, fallback Price) *Pricing {
+	table := make(map[string]Price, len(byModel))
+	for name, p := range byModel {
+		table[strings.ToLower(name)] = p
+	}
+	return &Pricing{byModel: table, fallback: fallback}
+}
+
+// For returns a model's price, or the fallback.
+func (p *Pricing) For(model string) Price {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	if v, ok := p.byModel[strings.ToLower(model)]; ok {
+		return v
+	}
+	return p.fallback
+}
+
+// AnythingPriced reports whether any model costs anything.
+//
+// A budget is spend per period, and spend is computed from a price table. With
+// no prices every request costs nothing, so a budget written on a team is a
+// number that can never move — displayed as a limit, reported as zero percent
+// used, and binding on nothing.
+//
+// Self-hosted models have no per-token price, so an empty table is the ordinary
+// case rather than a misconfiguration. What is not ordinary is showing somebody
+// a budget without saying so.
+func (p *Pricing) AnythingPriced() bool {
+	if p == nil {
+		return false
+	}
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+	if !p.fallback.Zero() {
+		return true
+	}
+	for _, price := range p.byModel {
+		if !price.Zero() {
+			return true
+		}
+	}
+	return false
+}

internal/accounting/limiter.go

@@ -0,0 +1,130 @@
+package accounting
+
+import (
+	"sync"
+	"time"
+)
+
+// Limiter enforces per-key rate limits.
+//
+// A sliding window rather than a fixed one. A fixed window lets a caller send
+// its whole minute's allowance in the last second of one window and again in
+// the first second of the next, so the observed rate is twice the configured
+// limit exactly when the system is least able to take it.
+//
+// The window is approximated from two buckets, which is the usual compromise:
+// exact enough that the doubling cannot happen, and bounded memory per key
+// rather than a timestamp per request.
+type Limiter struct {
+	now func() time.Time
+
+	mu   sync.Mutex
+	keys map[string]*window
+}
+
+type window struct {
+	// current and previous are counts in this minute and the one before.
+	current, previous int64
+	start             time.Time
+}
+
+// NewLimiter builds an empty limiter.
+func NewLimiter() *Limiter {
+	return &Limiter{now: time.Now, keys: map[string]*window{}}
+}
+
+const windowLen = time.Minute
+
+// Allow charges n against a key's limit and reports whether it fits.
+//
+// Charging and checking are one operation on purpose: doing them separately
+// lets two concurrent requests both see room and both take it.
+func (l *Limiter) Allow(key string, limit int, n int64) bool {
+	if limit <= 0 {
+		// No limit configured. Zero would be a limit of zero, which is why
+		// callers pass a nil pointer rather than 0 for "unset".
+		return true
+	}
+	now := l.now()
+
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	w := l.windowLocked(key, now)
+	if w.rate(now)+float64(n) > float64(limit) {
+		return false
+	}
+	w.current += n
+	return true
+}
+
+// Charge records usage that was already allowed, for reconciliation.
+//
+// Token limits are checked against an estimate before the request runs, because
+// the real figure only exists once it has. The difference is charged here, so a
+// caller whose requests are consistently larger than estimated is still bounded
+// — one window later than it would be with a perfect estimate.
+func (l *Limiter) Charge(key string, n int64) {
+	if n == 0 {
+		return
+	}
+	now := l.now()
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	w := l.windowLocked(key, now)
+	w.current += n
+	if w.current < 0 {
+		w.current = 0
+	}
+}
+
+// Rate returns a key's current observed rate.
+func (l *Limiter) Rate(key string) float64 {
+	now := l.now()
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.windowLocked(key, now).rate(now)
+}
+
+func (l *Limiter) windowLocked(key string, now time.Time) *window {
+	w, ok := l.keys[key]
+	if !ok {
+		w = &window{start: now.Truncate(windowLen)}
+		l.keys[key] = w
+	}
+	w.roll(now)
+	return w
+}
+
+// roll advances the window to now.
+func (w *window) roll(now time.Time) {
+	start := now.Truncate(windowLen)
+	switch {
+	case start.Equal(w.start):
+		return
+	case start.Sub(w.start) == windowLen:
+		w.previous, w.current = w.current, 0
+	default:
+		// More than one window has passed with no traffic.
+		w.previous, w.current = 0, 0
+	}
+	w.start = start
+}
+
+// rate weights the previous window by how much of it is still in view.
+func (w *window) rate(now time.Time) float64 {
+	elapsed := now.Sub(w.start)
+	if elapsed >= windowLen {
+		return float64(w.current)
+	}
+	remaining := 1 - float64(elapsed)/float64(windowLen)
+	return float64(w.previous)*remaining + float64(w.current)
+}
+
+// Forget drops a key's window. Used when a key is revoked; keeping it would
+// leak memory for every key that ever called.
+func (l *Limiter) Forget(key string) {
+	l.mu.Lock()
+	delete(l.keys, key)
+	l.mu.Unlock()
+}

internal/accounting/recorder.go

@@ -0,0 +1,189 @@
+package accounting
+
+import (
+	"context"
+	"log/slog"
+	"sync"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+// RecorderConfig tunes the batch writer.
+type RecorderConfig struct {
+	// FlushInterval and BatchSize decide when a batch is written. Whichever
+	// comes first.
+	FlushInterval time.Duration
+	BatchSize     int
+	// Capacity bounds the buffer. Beyond it, records are dropped rather than
+	// blocking a request — see Record.
+	Capacity int
+}
+
+// DefaultRecorderConfig returns sensible values.
+func DefaultRecorderConfig() RecorderConfig {
+	return RecorderConfig{FlushInterval: 200 * time.Millisecond, BatchSize: 500, Capacity: 8192}
+}
+
+func (c *RecorderConfig) normalize() {
+	d := DefaultRecorderConfig()
+	if c.FlushInterval <= 0 {
+		c.FlushInterval = d.FlushInterval
+	}
+	if c.BatchSize <= 0 {
+		c.BatchSize = d.BatchSize
+	}
+	if c.Capacity <= 0 {
+		c.Capacity = d.Capacity
+	}
+}
+
+// Toucher is the part of the authenticator the recorder flushes alongside
+// usage, so last-used timestamps ride the same transaction schedule.
+type Toucher interface {
+	FlushTouches(ctx context.Context) error
+}
+
+// Recorder batches usage records into the store.
+//
+// The buffered channel is the whole design. A request must never wait on a
+// database write, so Record is a non-blocking send and the writer drains it on
+// its own schedule.
+type Recorder struct {
+	cfg   RecorderConfig
+	store store.Store
+	touch Toucher
+	log   *slog.Logger
+
+	ch   chan store.Usage
+	done chan struct{}
+	wg   sync.WaitGroup
+
+	mu      sync.Mutex
+	dropped int64
+	written int64
+}
+
+// NewRecorder builds a recorder. Start begins writing.
+func NewRecorder(s store.Store, touch Toucher, cfg RecorderConfig, log *slog.Logger) *Recorder {
+	cfg.normalize()
+	if log == nil {
+		log = slog.Default()
+	}
+	return &Recorder{
+		cfg: cfg, store: s, touch: touch, log: log,
+		ch:   make(chan store.Usage, cfg.Capacity),
+		done: make(chan struct{}),
+	}
+}
+
+// Record queues one request's accounting.
+//
+// Never blocks. If the buffer is full — a database stalled long enough to fill
+// eight thousand records — the record is dropped and counted. Losing some
+// accounting is bad; making every request wait on a stalled database is worse,
+// because it converts a reporting problem into an outage.
+func (r *Recorder) Record(u store.Usage) {
+	select {
+	case r.ch <- u:
+	default:
+		r.mu.Lock()
+		r.dropped++
+		n := r.dropped
+		r.mu.Unlock()
+		if n == 1 || n%1000 == 0 {
+			r.log.Warn("dropping usage records: the store is not keeping up",
+				"dropped_total", n)
+		}
+	}
+}
+
+// Start runs the writer until ctx ends.
+func (r *Recorder) Start(ctx context.Context) {
+	r.wg.Add(1)
+	go func() {
+		defer r.wg.Done()
+		defer close(r.done)
+		r.run(ctx)
+	}()
+}
+
+// Stop waits for the writer to drain and finish.
+func (r *Recorder) Stop() { r.wg.Wait() }
+
+func (r *Recorder) run(ctx context.Context) {
+	t := time.NewTicker(r.cfg.FlushInterval)
+	defer t.Stop()
+
+	batch := make([]store.Usage, 0, r.cfg.BatchSize)
+	for {
+		select {
+		case <-ctx.Done():
+			// Drain what is already queued before leaving. These are completed
+			// requests that have already been served; throwing them away would
+			// mean a clean shutdown silently loses the last moments of billing.
+			r.drain(&batch)
+			r.flush(context.WithoutCancel(ctx), batch)
+			return
+
+		case u := <-r.ch:
+			batch = append(batch, u)
+			if len(batch) >= r.cfg.BatchSize {
+				r.flush(ctx, batch)
+				batch = batch[:0]
+			}
+
+		case <-t.C:
+			r.drain(&batch)
+			if len(batch) > 0 {
+				r.flush(ctx, batch)
+				batch = batch[:0]
+			}
+			if err := r.touchFlush(ctx); err != nil {
+				r.log.Warn("failed to write last-used timestamps", "err", err)
+			}
+		}
+	}
+}
+
+// drain moves everything queued into the batch without blocking.
+func (r *Recorder) drain(batch *[]store.Usage) {
+	for len(*batch) < r.cfg.BatchSize {
+		select {
+		case u := <-r.ch:
+			*batch = append(*batch, u)
+		default:
+			return
+		}
+	}
+}
+
+func (r *Recorder) flush(ctx context.Context, batch []store.Usage) {
+	if len(batch) == 0 {
+		return
+	}
+	if err := r.store.WriteUsage(ctx, batch); err != nil {
+		// The records are gone either way; what matters is that this is loud.
+		// Silently losing billing data is the kind of thing discovered at the
+		// end of a month.
+		r.log.Error("failed to write usage records", "count", len(batch), "err", err)
+		return
+	}
+	r.mu.Lock()
+	r.written += int64(len(batch))
+	r.mu.Unlock()
+}
+
+func (r *Recorder) touchFlush(ctx context.Context) error {
+	if r.touch == nil {
+		return nil
+	}
+	return r.touch.FlushTouches(ctx)
+}
+
+// Stats reports what the recorder has done, for metrics and tests.
+func (r *Recorder) Stats() (written, dropped int64) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.written, r.dropped
+}

internal/accounting/recorder_test.go

@@ -0,0 +1,210 @@
+package accounting
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+// fakeStore records what was written and can be made slow or broken.
+type fakeStore struct {
+	store.Store
+
+	mu      sync.Mutex
+	batches [][]store.Usage
+
+	block   chan struct{}
+	fail    atomic.Bool
+	touched atomic.Int64
+}
+
+func (f *fakeStore) WriteUsage(_ context.Context, rows []store.Usage) error {
+	if f.block != nil {
+		<-f.block
+	}
+	if f.fail.Load() {
+		return errors.New("store is down")
+	}
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.batches = append(f.batches, append([]store.Usage(nil), rows...))
+	return nil
+}
+
+func (f *fakeStore) rows() []store.Usage {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var out []store.Usage
+	for _, b := range f.batches {
+		out = append(out, b...)
+	}
+	return out
+}
+
+func (f *fakeStore) batchCount() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return len(f.batches)
+}
+
+type fakeToucher struct{ n *atomic.Int64 }
+
+func (f fakeToucher) FlushTouches(context.Context) error {
+	f.n.Add(1)
+	return nil
+}
+
+func TestRecordsAreBatchedNotWrittenOneByOne(t *testing.T) {
+	// The whole point of the buffer: a hundred requests must not be a hundred
+	// transactions.
+	f := &fakeStore{}
+	r := NewRecorder(f, nil, RecorderConfig{FlushInterval: 20 * time.Millisecond, BatchSize: 500},
+		slog.New(slog.DiscardHandler))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	r.Start(ctx)
+
+	for i := range 100 {
+		r.Record(store.Usage{KeyID: "k", PromptTokens: int64(i)})
+	}
+	waitFor(t, time.Second, func() bool { return len(f.rows()) == 100 })
+
+	if n := f.batchCount(); n > 5 {
+		t.Errorf("100 records took %d transactions", n)
+	}
+	cancel()
+	r.Stop()
+}
+
+func TestRecordNeverBlocksARequest(t *testing.T) {
+	// A stalled database must not become an outage. Losing accounting is bad;
+	// making every request wait on the store is worse.
+	f := &fakeStore{block: make(chan struct{})}
+	r := NewRecorder(f, nil, RecorderConfig{
+		FlushInterval: time.Millisecond, BatchSize: 1, Capacity: 8,
+	}, slog.New(slog.DiscardHandler))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	r.Start(ctx)
+	defer func() { close(f.block); cancel(); r.Stop() }()
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for range 1000 {
+			r.Record(store.Usage{KeyID: "k"})
+		}
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("Record blocked while the store was stalled")
+	}
+
+	_, dropped := r.Stats()
+	if dropped == 0 {
+		t.Error("the overflow was not counted, so nobody would ever know")
+	}
+}
+
+func TestAShutdownDoesNotLoseWhatIsAlreadyQueued(t *testing.T) {
+	// Those are completed requests that were already served. Throwing them away
+	// means a clean shutdown silently loses the last moments of billing.
+	f := &fakeStore{}
+	r := NewRecorder(f, nil, RecorderConfig{FlushInterval: time.Hour, BatchSize: 10_000},
+		slog.New(slog.DiscardHandler))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	r.Start(ctx)
+
+	for range 50 {
+		r.Record(store.Usage{KeyID: "k"})
+	}
+	cancel()
+	r.Stop()
+
+	if got := len(f.rows()); got != 50 {
+		t.Errorf("a clean shutdown wrote %d of 50 queued records", got)
+	}
+}
+
+func TestAFailedWriteIsLoudRatherThanSilent(t *testing.T) {
+	// Silently losing billing data is the kind of thing discovered at the end
+	// of a month.
+	f := &fakeStore{}
+	f.fail.Store(true)
+	logged := &countingHandler{}
+	r := NewRecorder(f, nil, RecorderConfig{FlushInterval: 10 * time.Millisecond, BatchSize: 1},
+		slog.New(logged))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	r.Start(ctx)
+	r.Record(store.Usage{KeyID: "k"})
+
+	waitFor(t, time.Second, func() bool { return logged.count(slog.LevelError) > 0 })
+	cancel()
+	r.Stop()
+
+	if written, _ := r.Stats(); written != 0 {
+		t.Errorf("a failed write was counted as %d written", written)
+	}
+}
+
+func TestLastUsedRidesTheSameSchedule(t *testing.T) {
+	var n atomic.Int64
+	f := &fakeStore{}
+	r := NewRecorder(f, fakeToucher{n: &n}, RecorderConfig{FlushInterval: 10 * time.Millisecond},
+		slog.New(slog.DiscardHandler))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	r.Start(ctx)
+	waitFor(t, time.Second, func() bool { return n.Load() > 0 })
+	cancel()
+	r.Stop()
+}
+
+type countingHandler struct {
+	mu     sync.Mutex
+	levels []slog.Level
+}
+
+func (c *countingHandler) Enabled(context.Context, slog.Level) bool { return true }
+func (c *countingHandler) WithAttrs([]slog.Attr) slog.Handler       { return c }
+func (c *countingHandler) WithGroup(string) slog.Handler            { return c }
+func (c *countingHandler) Handle(_ context.Context, r slog.Record) error {
+	c.mu.Lock()
+	c.levels = append(c.levels, r.Level)
+	c.mu.Unlock()
+	return nil
+}
+
+func (c *countingHandler) count(l slog.Level) int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	n := 0
+	for _, got := range c.levels {
+		if got == l {
+			n++
+		}
+	}
+	return n
+}
+
+func waitFor(t *testing.T, limit time.Duration, pred func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(limit)
+	for time.Now().Before(deadline) {
+		if pred() {
+			return
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	t.Fatal("condition never became true")
+}

internal/agent/agent_test.go

@@ -0,0 +1,758 @@
+package agent
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"testing"
+	"time"
+
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
+)
+
+// statusRecorder captures what the control plane hears about shards, and what
+// it is asked to adopt.
+type statusRecorder struct {
+	ch chan status
+
+	mu      sync.Mutex
+	calls   []adoptCall
+	adopted []reconciler.AdoptedShard
+	unclaim []string // shard ids to report back as unclaimed
+	fetches []fetchReport
+}
+
+type status struct {
+	replicaID, shardID string
+	state              supervisor.State
+	err                string
+}
+
+func newRecorder() *statusRecorder { return &statusRecorder{ch: make(chan status, 128)} }
+
+func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
+	select {
+	case r.ch <- status{replicaID, shardID, st, errMsg}:
+	default:
+	}
+}
+
+// fetches records download progress, so a test can assert that a fetch is
+// visible from the control plane rather than only in the node's log.
+func (r *statusRecorder) OnFetchProgress(replicaID, shardID, source string, done, total int64) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.fetches = append(r.fetches, fetchReport{replicaID, shardID, source, done, total})
+}
+
+func (r *statusRecorder) fetchReports() []fetchReport {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return append([]fetchReport(nil), r.fetches...)
+}
+
+type fetchReport struct {
+	replicaID, shardID, source string
+	done, total                int64
+}
+
+func (r *statusRecorder) Adopt(nodeID string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.calls = append(r.calls, adoptCall{nodeID: nodeID, shards: shards})
+	r.adopted = append(r.adopted, shards...)
+
+	var unclaimed []reconciler.AdoptedShard
+	for _, s := range shards {
+		if slices.Contains(r.unclaim, s.ShardID) {
+			unclaimed = append(unclaimed, s)
+		}
+	}
+	return unclaimed
+}
+
+// adoptCall records that the service reached the reconciler at all, which an
+// empty shard list used to prevent.
+type adoptCall struct {
+	nodeID string
+	shards []reconciler.AdoptedShard
+}
+
+func (r *statusRecorder) adoptCalls() []adoptCall {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return slices.Clone(r.calls)
+}
+
+func (r *statusRecorder) adoptedShards() []reconciler.AdoptedShard {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return slices.Clone(r.adopted)
+}
+
+func (r *statusRecorder) markUnclaimed(ids ...string) {
+	r.mu.Lock()
+	r.unclaim = append(r.unclaim, ids...)
+	r.mu.Unlock()
+}
+
+func (r *statusRecorder) await(t *testing.T, limit time.Duration, pred func(status) bool) status {
+	t.Helper()
+	deadline := time.After(limit)
+	var seen []status
+	for {
+		select {
+		case s := <-r.ch:
+			seen = append(seen, s)
+			if pred(s) {
+				return s
+			}
+		case <-deadline:
+			t.Fatalf("no matching status within %v; saw %+v", limit, seen)
+			return status{}
+		}
+	}
+}
+
+// fakeEngineScript writes a script that stands in for vLLM.
+//
+// It parses the real --port flag the vLLM adapter generates rather than being
+// told out of band, so the adapter's argument construction is exercised too.
+func fakeEngineScript(t *testing.T, body string) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "fake-vllm")
+	script := `#!/bin/sh
+port=""
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --port) port="$2"; shift 2 ;;
+    *) shift ;;
+  esac
+done
+` + body + "\n"
+	if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	return path
+}
+
+// servingEngine answers /health and /v1/models, which is what readiness needs.
+func servingEngine(t *testing.T) string {
+	t.Helper()
+	py, err := exec.LookPath("python3")
+	if err != nil {
+		t.Skip("python3 is required for the fake engine")
+	}
+	return fakeEngineScript(t, `exec `+py+` -c '
+import http.server, json, sys
+port = int(sys.argv[1])
+class H(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path == "/health":
+            self.send_response(200); self.end_headers(); return
+        if self.path == "/v1/models":
+            b = json.dumps({"object":"list","data":[{"id":"test-model"}]}).encode()
+            self.send_response(200)
+            self.send_header("content-type","application/json")
+            self.send_header("content-length", str(len(b)))
+            self.end_headers(); self.wfile.write(b); return
+        self.send_response(404); self.end_headers()
+    def log_message(self, *a): pass
+http.server.HTTPServer(("127.0.0.1", port), H).serve_forever()
+' "$port"`)
+}
+
+type harness struct {
+	svc *agentsvc.Service
+	inv *inventory.Store
+	rec *statusRecorder
+	srv *httptest.Server
+	ag  *Agent
+	ln  *dropListener
+	// streams counts how many times an agent has opened the Connect stream,
+	// which is the only honest way to tell a reconnect from a stale entry.
+	streams *atomic.Int64
+}
+
+// dropListener tracks accepted connections so a test can sever them.
+//
+// httptest's CloseClientConnections does not work here: h2c with prior
+// knowledge hijacks the connection, and httptest stops tracking hijacked
+// connections. A test relying on it would pass without ever reconnecting.
+type dropListener struct {
+	net.Listener
+	mu    sync.Mutex
+	conns []net.Conn
+}
+
+func (l *dropListener) Accept() (net.Conn, error) {
+	c, err := l.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	l.mu.Lock()
+	l.conns = append(l.conns, c)
+	l.mu.Unlock()
+	return c, nil
+}
+
+// drop severs every connection, as a control plane restart would.
+func (l *dropListener) drop() {
+	l.mu.Lock()
+	conns := l.conns
+	l.conns = nil
+	l.mu.Unlock()
+	for _, c := range conns {
+		_ = c.Close()
+	}
+}
+
+// start wires a real agent to a real service over h2c, so this exercises the
+// protocol rather than two structs calling each other.
+func start(t *testing.T, cfgFns ...func(*Config)) *harness {
+	t.Helper()
+	return startWith(t, slog.New(slog.DiscardHandler), cfgFns...)
+}
+
+// startLogged is start with logging on, for diagnosing protocol behaviour.
+func startLogged(t *testing.T, cfgFns ...func(*Config)) *harness {
+	t.Helper()
+	return startWith(t, slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})), cfgFns...)
+}
+
+func startWith(t *testing.T, log *slog.Logger, cfgFns ...func(*Config)) *harness {
+	t.Helper()
+
+	inv := inventory.NewStore()
+	rec := newRecorder()
+	svc := agentsvc.New(inv, rec, false, log)
+
+	path, handler := svc.Handler()
+	mux := http.NewServeMux()
+	mux.Handle(path, handler)
+
+	var streams atomic.Int64
+	counting := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		streams.Add(1)
+		mux.ServeHTTP(w, r)
+	})
+
+	raw, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ln := &dropListener{Listener: raw}
+	// connect's bidirectional streams need HTTP/2; h2c provides it without TLS.
+	srv := httptest.NewUnstartedServer(h2c.NewHandler(counting, &http2.Server{}))
+	srv.Listener = ln
+	srv.Start()
+
+	cfg := DefaultConfig()
+	cfg.NodeID = "n1"
+	cfg.Server = srv.URL
+	cfg.MgmtAddr = "127.0.0.1"
+	cfg.HeartbeatInterval = 50 * time.Millisecond
+	cfg.ReconnectBase = 20 * time.Millisecond
+	cfg.ReconnectMax = 100 * time.Millisecond
+	cfg.Discovery.SkipGPUs = true
+	cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent")
+	cfg.Superv = supervisor.Config{
+		WorkDir: t.TempDir(), LogLines: 64,
+		StopTimeout: 2 * time.Second, HealthInterval: 20 * time.Millisecond,
+	}
+	for _, f := range cfgFns {
+		f(&cfg)
+	}
+
+	client := &http.Client{Transport: &http2.Transport{
+		AllowHTTP: true,
+		DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
+			return (&net.Dialer{}).DialContext(ctx, network, addr)
+		},
+	}}
+
+	ag, err := New(cfg, client, log)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() { _ = ag.Run(ctx) }()
+
+	h := &harness{svc: svc, inv: inv, rec: rec, srv: srv, ag: ag, ln: ln, streams: &streams}
+	t.Cleanup(func() {
+		cancel()
+		ag.Supervisor().StopAll(context.Background())
+		srv.Close()
+	})
+	h.waitRegistered(t)
+	return h
+}
+
+func (h *harness) waitRegistered(t *testing.T) {
+	t.Helper()
+	deadline := time.Now().Add(10 * time.Second)
+	for time.Now().Before(deadline) {
+		if _, ok := h.inv.Get("n1"); ok && len(h.svc.ConnectedNodes()) == 1 {
+			return
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	t.Fatal("the agent never registered")
+}
+
+// shardCommand builds a single-node vLLM shard on a free port.
+//
+// The model path is a real file. It used to be "/models/x", which stopped
+// working once the agent began checking that weights exist before launching an
+// engine — the check doing exactly what it is for, on a test that was relying
+// on nobody looking.
+func shardCommand(t *testing.T, shardID string) reconciler.ShardCommand {
+	t.Helper()
+	modelPath := filepath.Join(t.TempDir(), "model.gguf")
+	if err := os.WriteFile(modelPath, []byte("weights"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return reconciler.ShardCommand{
+		ReplicaID: "rep-1", ShardID: shardID, NodeID: "n1",
+		Spec: engines.ShardSpec{
+			ReplicaID: "rep-1", ShardID: shardID, Rank: 0, NNodes: 1,
+			GPUIndices: []int{0}, HTTPPort: freePort(t),
+			Profile: engines.Profile{
+				Name: "p", Engine: "vllm", TP: 1,
+				ModelPath: modelPath, ServedModelName: "test-model",
+			},
+		},
+		StartupTimeout: 15 * time.Second,
+	}
+}
+
+func freePort(t *testing.T) int {
+	t.Helper()
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+	return ln.Addr().(*net.TCPAddr).Port
+}
+
+func TestRegistrationCarriesInventory(t *testing.T) {
+	// The node must actually have the adapters it declares. It used to declare
+	// mlx5_0 and mlx5_1 on a node whose sysfs held neither, which passed
+	// because nothing compared the two — and that is the gap a real cluster
+	// would have found as a replica hanging on its first all-reduce.
+	ibRoot := ibTreeWithActivePorts(t, "mlx5_0", "mlx5_1")
+	h := start(t, func(c *Config) {
+		c.Hostname = "gpu-a-01"
+		c.Labels = map[string]string{"rack": "r12"}
+		c.Discovery.IBRoot = ibRoot
+		c.Fabrics = []inventory.FabricAttachment{
+			{Fabric: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}, RateGbps: 200},
+		}
+	})
+
+	n, ok := h.inv.Get("n1")
+	if !ok {
+		t.Fatal("node missing from inventory")
+	}
+	if n.Hostname != "gpu-a-01" || n.MgmtAddr != "127.0.0.1" {
+		t.Errorf("identity wrong: %+v", n)
+	}
+	if n.Labels["rack"] != "r12" {
+		t.Errorf("labels lost: %v", n.Labels)
+	}
+	// The fabric label cannot be inferred from the wire, so losing it in
+	// transit would silently disable multi-node placement.
+	if !n.HasFabric("ib-a") {
+		t.Fatalf("fabric attachment lost: %+v", n.Fabrics)
+	}
+	if got := n.HCAsFor("ib-a"); !slices.Equal(got, []string{"mlx5_0", "mlx5_1"}) {
+		t.Errorf("HCAs=%v", got)
+	}
+}
+
+func TestHeartbeatKeepsNodeAlive(t *testing.T) {
+	h := start(t)
+
+	n, _ := h.inv.Get("n1")
+	first := n.LastSeen
+
+	time.Sleep(200 * time.Millisecond)
+	n, _ = h.inv.Get("n1")
+	if !n.LastSeen.After(first) {
+		t.Error("heartbeats are not refreshing LastSeen; the node would be declared lost")
+	}
+}
+
+func TestStartAndStopShardEndToEnd(t *testing.T) {
+	h := start(t, func(c *Config) {
+		c.Engines.VLLM.Bin = servingEngine(t)
+	})
+
+	cmd := shardCommand(t, "shard-a")
+	if err := h.svc.StartShard(t.Context(), cmd); err != nil {
+		t.Fatal(err)
+	}
+
+	got := h.rec.await(t, 30*time.Second, func(s status) bool {
+		return s.shardID == "shard-a" && s.state == supervisor.Ready
+	})
+	if got.replicaID != "rep-1" {
+		t.Errorf("replica id lost in transit: %q", got.replicaID)
+	}
+
+	if err := h.svc.StopShard(t.Context(), "n1", "shard-a"); err != nil {
+		t.Fatal(err)
+	}
+	h.rec.await(t, 15*time.Second, func(s status) bool {
+		return s.shardID == "shard-a" && s.state == supervisor.Stopped
+	})
+}
+
+func TestFailedShardCarriesDiagnosis(t *testing.T) {
+	// When a model fails to load the engine's output is the only diagnosis, so
+	// it has to survive the trip to the control plane.
+	h := start(t, func(c *Config) {
+		c.Engines.VLLM.Bin = fakeEngineScript(t, `echo 'CUDA out of memory' >&2; exit 1`)
+	})
+
+	if err := h.svc.StartShard(t.Context(), shardCommand(t, "shard-bad")); err != nil {
+		t.Fatal(err)
+	}
+	got := h.rec.await(t, 30*time.Second, func(s status) bool {
+		return s.shardID == "shard-bad" && s.state == supervisor.Failed
+	})
+	if got.err == "" {
+		t.Error("the failure carried no error message")
+	}
+}
+
+func TestEngineThatNeverBecomesReady(t *testing.T) {
+	// A process that runs but never serves must time out rather than hang the
+	// replica forever.
+	h := start(t, func(c *Config) {
+		c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 60`)
+	})
+
+	cmd := shardCommand(t, "shard-stuck")
+	cmd.StartupTimeout = 500 * time.Millisecond
+	if err := h.svc.StartShard(t.Context(), cmd); err != nil {
+		t.Fatal(err)
+	}
+	h.rec.await(t, 20*time.Second, func(s status) bool {
+		return s.shardID == "shard-stuck" && s.state == supervisor.Failed
+	})
+}
+
+func TestValidationRejectsBeforeLaunch(t *testing.T) {
+	// An engine that refuses a profile does so after minutes of loading;
+	// catching it at dispatch costs nothing.
+	h := start(t, func(c *Config) {
+		c.Engines.VLLM.Bin = servingEngine(t)
+	})
+
+	cmd := shardCommand(t, "shard-invalid")
+	cmd.Spec.Profile.ModelPath = "" // validation requires one
+
+	if err := h.svc.StartShard(t.Context(), cmd); err != nil {
+		t.Fatal(err)
+	}
+	got := h.rec.await(t, 15*time.Second, func(s status) bool {
+		return s.shardID == "shard-invalid" && s.state == supervisor.Failed
+	})
+	if got.err == "" {
+		t.Error("validation failure carried no reason")
+	}
+}
+
+func TestDispatchToUnconnectedNode(t *testing.T) {
+	h := start(t)
+
+	err := h.svc.StartShard(t.Context(), reconciler.ShardCommand{NodeID: "nowhere", ShardID: "x"})
+	if err == nil {
+		t.Fatal("dispatching to an unconnected node succeeded")
+	}
+	if !errors.Is(err, agentsvc.ErrNotConnected) {
+		t.Errorf("err=%v, expected ErrNotConnected", err)
+	}
+}
+
+func TestAgentReconnects(t *testing.T) {
+	// A control plane restart must not require touching every GPU node.
+	h := start(t)
+
+	before := h.streams.Load()
+	h.ln.drop()
+
+	// Counting streams, not connected nodes: a stale entry in the connection
+	// map would make a node look connected without any reconnect having
+	// happened, and an assertion on it would pass vacuously.
+	deadline := time.Now().Add(15 * time.Second)
+	for time.Now().Before(deadline) {
+		if h.streams.Load() > before && len(h.svc.ConnectedNodes()) == 1 {
+			if err := h.svc.DrainShard(t.Context(), "n1", "nothing"); err != nil {
+				t.Fatalf("the reconnected stream does not accept commands: %v", err)
+			}
+			return
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	t.Fatalf("the agent never reconnected (streams %d -> %d)", before, h.streams.Load())
+}
+
+func TestIdempotentStart(t *testing.T) {
+	// The control plane retries commands; a repeat must not start a second
+	// process (plan §5.6).
+	h := start(t, func(c *Config) {
+		c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 30`)
+	})
+
+	cmd := shardCommand(t, "shard-idem")
+	if err := h.svc.StartShard(t.Context(), cmd); err != nil {
+		t.Fatal(err)
+	}
+	h.rec.await(t, 15*time.Second, func(s status) bool {
+		return s.shardID == "shard-idem" && s.state == supervisor.Starting
+	})
+
+	sup := h.ag.Supervisor()
+	sh, err := sup.Get("shard-idem")
+	if err != nil {
+		t.Fatal(err)
+	}
+	pid := sh.Status().PID
+
+	if err := h.svc.StartShard(t.Context(), cmd); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(400 * time.Millisecond)
+
+	sh2, err := sup.Get("shard-idem")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if sh2.Status().PID != pid {
+		t.Errorf("a second process was started: pid %d then %d", pid, sh2.Status().PID)
+	}
+}
+
+// TestRunningShardsReportedOnReconnect covers the control plane restart hole
+// (plan §6.5): a reconnecting agent has to describe what it is already running,
+// or the control plane concludes the cluster is empty and places duplicates
+// while the originals keep holding GPUs.
+func TestRunningShardsReportedOnReconnect(t *testing.T) {
+	h := start(t, func(c *Config) {
+		c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 60`)
+	})
+
+	cmd := shardCommand(t, "shard-live")
+	cmd.Pool = "ib-a/h100-80g"
+	cmd.Fabric = "ib-a"
+	cmd.Spec.HCAs = []string{"mlx5_0"}
+
+	if err := h.svc.StartShard(t.Context(), cmd); err != nil {
+		t.Fatal(err)
+	}
+	h.rec.await(t, 15*time.Second, func(s status) bool {
+		return s.shardID == "shard-live" && s.state == supervisor.Starting
+	})
+
+	// Drop the stream, as a control plane restart would.
+	h.ln.drop()
+
+	deadline := time.Now().Add(15 * time.Second)
+	for time.Now().Before(deadline) {
+		if got := h.rec.adoptedShards(); len(got) > 0 {
+			a := got[0]
+			if a.ShardID != "shard-live" || a.ReplicaID != "rep-1" {
+				t.Fatalf("wrong shard reported: %+v", a)
+			}
+			// Everything the control plane needs to rebuild the replica.
+			if a.Model != "test-model" || a.Profile != "p" {
+				t.Errorf("catalog identity lost: model=%q profile=%q", a.Model, a.Profile)
+			}
+			if a.Pool != "ib-a/h100-80g" || a.Fabric != "ib-a" {
+				t.Errorf("pool/fabric lost: %q %q", a.Pool, a.Fabric)
+			}
+			if !slices.Equal(a.HCAs, []string{"mlx5_0"}) {
+				t.Errorf("HCAs lost: %v", a.HCAs)
+			}
+			if a.Endpoint == "" {
+				t.Error("rank 0 reported no endpoint; the router would have nowhere to send")
+			}
+			if len(a.GPUIndices) == 0 {
+				t.Error("GPU indices lost; the inventory could not reclaim them")
+			}
+			return
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	t.Fatal("the reconnecting agent never reported its running shard")
+}
+
+// TestUnclaimedShardIsStopped covers the other half: a shard whose model left
+// the catalog is holding GPUs nothing will ever route to.
+func TestUnclaimedShardIsStopped(t *testing.T) {
+	h := start(t, func(c *Config) {
+		c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 60`)
+	})
+
+	cmd := shardCommand(t, "shard-orphaned")
+	if err := h.svc.StartShard(t.Context(), cmd); err != nil {
+		t.Fatal(err)
+	}
+	h.rec.await(t, 15*time.Second, func(s status) bool {
+		return s.shardID == "shard-orphaned" && s.state == supervisor.Starting
+	})
+
+	sh, err := h.ag.Supervisor().Get("shard-orphaned")
+	if err != nil {
+		t.Fatal(err)
+	}
+	pid := sh.Status().PID
+
+	// The control plane will refuse to claim it on reconnect.
+	h.rec.markUnclaimed("shard-orphaned")
+	h.ln.drop()
+
+	deadline := time.Now().Add(20 * time.Second)
+	for time.Now().Before(deadline) {
+		if !processAliveForTest(pid) {
+			return
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	t.Error("an unclaimed shard kept running and holding its GPUs")
+}
+
+// processAliveForTest mirrors the supervisor's liveness check.
+func processAliveForTest(pid int) bool {
+	return syscall.Kill(pid, 0) == nil
+}
+
+func TestAShardWithMissingWeightsFailsBeforeLaunching(t *testing.T) {
+	// The commonest setup mistake on a fresh node. Left to the engine it
+	// surfaces minutes into a load, in the engine's own words, on a machine
+	// nobody is watching — so it is caught here, where the message can say
+	// what is actually wrong.
+	a := &Agent{cfg: Config{WeightsCache: t.TempDir()}, log: slog.New(slog.DiscardHandler)}
+	_, err := a.weights().Ensure(t.Context(), filepath.Join(t.TempDir(), "not-there.gguf"))
+	if err == nil {
+		t.Fatal("a missing weights file was accepted")
+	}
+	if !strings.Contains(err.Error(), "not-there.gguf") {
+		t.Errorf("the error does not name the file: %v", err)
+	}
+}
+
+func TestTheCacheDefaultsToTheFirstModelDirectory(t *testing.T) {
+	// A node that has said where its models live has already answered this;
+	// asking twice is a second thing to get out of step.
+	dir := t.TempDir()
+	a := &Agent{cfg: Config{ModelDirs: []string{dir, "/elsewhere"}}, log: slog.New(slog.DiscardHandler)}
+	if got := a.weights().CacheDir; got != dir {
+		t.Errorf("cache = %q, want the first model directory %q", got, dir)
+	}
+}
+
+// ibTreeWithActivePorts builds a sysfs InfiniBand tree with the named adapters,
+// each with one active port, shaped like the real thing.
+func ibTreeWithActivePorts(t *testing.T, names ...string) string {
+	t.Helper()
+	root := t.TempDir()
+	for _, name := range names {
+		dir := filepath.Join(root, name, "ports", "1")
+		if err := os.MkdirAll(filepath.Join(dir, "gids"), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		for file, content := range map[string]string{
+			"state":      "4: ACTIVE",
+			"phys_state": "5: LinkUp",
+			"rate":       "200 Gb/sec (4X HDR)",
+			"link_layer": "InfiniBand",
+			"lid":        "0x1a",
+			"gids/0":     "fec0:0000:0000:000a:0002:c903:00a0:1234",
+		} {
+			if err := os.WriteFile(filepath.Join(dir, file), []byte(content+"\n"), 0o644); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	return root
+}
+
+func TestAFabricWhoseAdaptersAreAbsentIsNotAdvertised(t *testing.T) {
+	// The failure this prevents: a typo, or a card that enumerated differently
+	// after a reboot, produced NCCL_IB_HCA naming an adapter that is not there.
+	// The replica starts, passes its health checks, and hangs on its first
+	// all-reduce — the exact case the fabric invariant exists to prevent,
+	// arriving through a door nobody was watching.
+	h := start(t, func(c *Config) {
+		c.Discovery.IBRoot = ibTreeWithActivePorts(t, "mlx5_0")
+		c.Fabrics = []inventory.FabricAttachment{
+			{Fabric: "ib-typo", HCAs: []string{"mlx5_9"}, RateGbps: 200},
+		}
+	})
+
+	n, ok := h.inv.Get("n1")
+	if !ok {
+		t.Fatal("node missing from inventory")
+	}
+	if n.HasFabric("ib-typo") {
+		t.Errorf("a fabric with no usable adapter was advertised: %+v", n.Fabrics)
+	}
+}
+
+func TestAFabricRateIsTheSlowestActiveLink(t *testing.T) {
+	// A collective runs at the speed of its slowest link, so reporting the
+	// fastest would have the scheduler believe a fabric is quicker than it is.
+	root := t.TempDir()
+	for name, rate := range map[string]string{"mlx5_0": "200", "mlx5_1": "100"} {
+		dir := filepath.Join(root, name, "ports", "1")
+		if err := os.MkdirAll(filepath.Join(dir, "gids"), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		for file, content := range map[string]string{
+			"state": "4: ACTIVE", "phys_state": "5: LinkUp",
+			"rate": rate + " Gb/sec (4X HDR)", "link_layer": "InfiniBand",
+			"lid": "0x1a", "gids/0": "fec0::1",
+		} {
+			if err := os.WriteFile(filepath.Join(dir, file), []byte(content+"\n"), 0o644); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	h := start(t, func(c *Config) {
+		c.Discovery.IBRoot = root
+		c.Fabrics = []inventory.FabricAttachment{
+			{Fabric: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}},
+		}
+	})
+
+	n, _ := h.inv.Get("n1")
+	for _, f := range n.Fabrics {
+		if f.Fabric == "ib-a" && f.RateGbps != 100 {
+			t.Errorf("fabric rate = %d, want the slower link's 100", f.RateGbps)
+		}
+	}
+}

internal/agent/discovery/discover.go

@@ -0,0 +1,88 @@
+package discovery
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+)
+
+// ErrNoNVML means this binary was built without the NVML backend.
+//
+// It is declared here rather than in the build-tagged file so callers — and
+// tests — can refer to it in either build.
+var ErrNoNVML = errors.New("built without the nvml tag; rebuild with -tags nvml for NVLink topology")
+
+// Options configure a full inventory scan.
+type Options struct {
+	// NvidiaSMI is the path to nvidia-smi; empty means look it up on PATH.
+	// Worth configuring on nodes where the driver lives outside the default
+	// paths, which containerized driver installs often do.
+	NvidiaSMI string `koanf:"nvidia_smi"`
+	// IBRoot is the sysfs InfiniBand directory.
+	IBRoot string `koanf:"ib_root"`
+	// PreferNVML uses the NVML backend when this binary has one. It is the only
+	// source of NVLink topology.
+	PreferNVML bool `koanf:"prefer_nvml"`
+	// SkipGPUs omits GPU discovery. A node without nvidia-smi is already
+	// found to have none, so this is for one whose driver is broken.
+	SkipGPUs bool `koanf:"skip_gpus"`
+	// ModelDirs are scanned for weights. Empty means do not look.
+	ModelDirs []string `koanf:"-"`
+}
+
+// Discover builds the node's inventory.
+//
+// When NVML is preferred but fails, it falls back to nvidia-smi and says so in
+// Inventory.Source. The fallback is logged rather than silent: on a node that
+// is supposed to have working NVML, a quiet downgrade hides a broken driver and
+// costs the scheduler its NVLink topology.
+func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, error) {
+	if log == nil {
+		log = slog.Default()
+	}
+	inv := Inventory{Source: "none"}
+	// First, because it has nothing to do with GPUs and every path below can
+	// return early. Read after them, a node with no nvidia-smi reported no
+	// memory at all: GPU discovery fails, Discover returns, and the fact that
+	// has nothing to do with cards goes with it. That is exactly the node whose
+	// memory matters most, since a CPU profile is what it will be asked to run.
+	inv.MemoryTotalMB = MemoryTotalMB()
+	// Also before the GPU paths, and for the same reason: which weights a node
+	// holds has nothing to do with its cards, and a node with none is exactly
+	// where a CPU profile will look for them.
+	inv.Weights = ScanWeights(opts.ModelDirs)
+	// And the disk they sit on. A node can hold a model today and be unable to
+	// take the next one, which the file list alone never shows.
+	inv.Storage = ScanStorage(opts.ModelDirs)
+
+	if !opts.SkipGPUs {
+		switch {
+		case opts.PreferNVML && nvmlAvailable:
+			gpus, err := DiscoverGPUsNVML(ctx)
+			if err == nil {
+				inv.GPUs, inv.Source = gpus, "nvml"
+				break
+			}
+			log.Warn("NVML discovery failed, falling back to nvidia-smi; "+
+				"NVLink topology will be unavailable", "err", err)
+			fallthrough
+		default:
+			gpus, err := DiscoverGPUsSMI(ctx, opts.NvidiaSMI)
+			if err != nil {
+				return inv, fmt.Errorf("gpu discovery: %w", err)
+			}
+			inv.GPUs, inv.Source = gpus, "nvidia-smi"
+		}
+	}
+
+	hcas, err := DiscoverIB(opts.IBRoot)
+	if err != nil {
+		return inv, fmt.Errorf("infiniband discovery: %w", err)
+	}
+	inv.HCAs = hcas
+	return inv, nil
+}
+
+// NVMLSupported reports whether this binary includes the NVML backend.
+func NVMLSupported() bool { return nvmlAvailable }

internal/agent/discovery/discover_test.go

@@ -0,0 +1,119 @@
+package discovery
+
+import (
+	"context"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+func fakeSMI(t *testing.T) string {
+	t.Helper()
+	stub := filepath.Join(t.TempDir(), "nvidia-smi")
+	script := "#!/bin/sh\ncat <<'EOF'\n" + smiSample + "EOF\n"
+	if err := os.WriteFile(stub, []byte(script), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	return stub
+}
+
+func TestDiscoverUsesSMIByDefault(t *testing.T) {
+	inv, err := Discover(t.Context(), Options{
+		NvidiaSMI: fakeSMI(t),
+		IBRoot:    filepath.Join(t.TempDir(), "absent"),
+	}, slog.New(slog.DiscardHandler))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if inv.Source != "nvidia-smi" {
+		t.Errorf("source=%q", inv.Source)
+	}
+	if len(inv.GPUs) != 2 {
+		t.Errorf("got %d GPUs", len(inv.GPUs))
+	}
+}
+
+func TestDiscoverFallsBackWhenNVMLUnavailable(t *testing.T) {
+	// Asking for NVML in a build without it must still produce an inventory:
+	// the fallback is the whole point of the build tag.
+	inv, err := Discover(t.Context(), Options{
+		PreferNVML: true,
+		NvidiaSMI:  fakeSMI(t),
+		IBRoot:     filepath.Join(t.TempDir(), "absent"),
+	}, slog.New(slog.DiscardHandler))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(inv.GPUs) == 0 {
+		t.Fatal("no GPUs discovered through the fallback")
+	}
+	if NVMLSupported() {
+		// On an NVML build without a driver the NVML attempt fails and the
+		// fallback still has to work.
+		if inv.Source != "nvml" && inv.Source != "nvidia-smi" {
+			t.Errorf("source=%q", inv.Source)
+		}
+	} else if inv.Source != "nvidia-smi" {
+		t.Errorf("a build without NVML reported source=%q", inv.Source)
+	}
+}
+
+func TestDiscoverSkipGPUs(t *testing.T) {
+	inv, err := Discover(t.Context(), Options{
+		SkipGPUs: true,
+		IBRoot:   fakeSysfs(t),
+	}, slog.New(slog.DiscardHandler))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(inv.GPUs) != 0 {
+		t.Errorf("GPUs were discovered despite SkipGPUs")
+	}
+	if len(inv.HCAs) != 3 {
+		t.Errorf("got %d HCAs", len(inv.HCAs))
+	}
+	if inv.Source != "none" {
+		t.Errorf("source=%q", inv.Source)
+	}
+}
+
+func TestDiscoverReportsGPUFailure(t *testing.T) {
+	// A node meant to have GPUs whose discovery fails must say so rather than
+	// reporting an empty, healthy-looking inventory.
+	_, err := Discover(context.Background(), Options{
+		NvidiaSMI: filepath.Join(t.TempDir(), "absent"),
+		IBRoot:    filepath.Join(t.TempDir(), "absent"),
+	}, slog.New(slog.DiscardHandler))
+	if err == nil {
+		t.Error("a failed GPU discovery was reported as success")
+	}
+}
+
+func TestNVMLSupportedMatchesBuild(t *testing.T) {
+	// Documents which backend this binary has, and keeps both files compiling.
+	t.Logf("NVML backend compiled in: %v", NVMLSupported())
+	if !NVMLSupported() {
+		if _, err := DiscoverGPUsNVML(t.Context()); err != ErrNoNVML {
+			t.Errorf("expected ErrNoNVML, got %v", err)
+		}
+	}
+}
+
+func TestMemoryIsReportedEvenWhenGPUDiscoveryFails(t *testing.T) {
+	// A node with no nvidia-smi is exactly the node whose memory matters, since
+	// a CPU profile is what it will be asked to run. Reading memory after GPU
+	// discovery meant that node reported none: the GPU error returns early and
+	// takes a fact that has nothing to do with cards with it. Seen on node05,
+	// which has 32 GB and was reporting "-".
+	inv, err := Discover(t.Context(), Options{NvidiaSMI: "/nonexistent/nvidia-smi"}, nil)
+	if err == nil {
+		t.Fatal("GPU discovery was expected to fail with no nvidia-smi")
+	}
+	// On Linux this is a real number; elsewhere there is no /proc/meminfo and 0
+	// is the honest answer. Either way it must not be lost to the GPU error.
+	if runtime.GOOS == "linux" && inv.MemoryTotalMB <= 0 {
+		t.Errorf("memory was not reported alongside a GPU failure: %d", inv.MemoryTotalMB)
+	}
+}

internal/agent/discovery/fabrics.go

@@ -0,0 +1,116 @@
+package discovery
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// FabricCheck is one declared fabric measured against what the node actually
+// has.
+type FabricCheck struct {
+	Fabric string
+	// Usable are the declared adapters that exist and have an active port.
+	Usable []string
+	// RateGbps is the slowest active port among them, because a collective
+	// runs at the speed of its slowest link.
+	RateGbps int
+	// Problems are the declared adapters that cannot carry traffic, and why.
+	Problems []string
+}
+
+// OK reports whether this fabric can carry a collective at all.
+func (c FabricCheck) OK() bool { return len(c.Usable) > 0 }
+
+// CheckFabrics measures declared fabric membership against discovered adapters.
+//
+// Fabric membership has to be configuration — OpenSM's default subnet prefix is
+// the same on every fabric, so a matching prefix proves nothing — but taking it
+// entirely on trust was a mistake of a different kind. A typo, or a card that
+// enumerated differently after a reboot, produced NCCL_IB_HCA naming an adapter
+// that does not exist; and an adapter whose cable is out, or whose subnet
+// manager is down, is present in sysfs and carries nothing.
+//
+// Either way the replica starts, passes its health checks, and hangs on its
+// first all-reduce — which is the failure the whole fabric invariant exists to
+// prevent, arriving through a door nobody was watching. Declared and real are
+// now compared where both are visible, which is only here.
+func CheckFabrics(declared map[string][]string, discovered []HCA) []FabricCheck {
+	byName := make(map[string]HCA, len(discovered))
+	for _, h := range discovered {
+		byName[h.Name] = h
+	}
+
+	names := make([]string, 0, len(declared))
+	for f := range declared {
+		names = append(names, f)
+	}
+	sort.Strings(names)
+
+	out := make([]FabricCheck, 0, len(names))
+	for _, fabric := range names {
+		c := FabricCheck{Fabric: fabric}
+		for _, want := range declared[fabric] {
+			hca, found := byName[want]
+			if !found {
+				c.Problems = append(c.Problems, fmt.Sprintf(
+					"%s is not on this node (found: %s)", want, listNames(discovered)))
+				continue
+			}
+			rate, active := activeRate(hca)
+			if !active {
+				c.Problems = append(c.Problems, fmt.Sprintf(
+					"%s has no active port (%s)", want, portStates(hca)))
+				continue
+			}
+			c.Usable = append(c.Usable, want)
+			if c.RateGbps == 0 || rate < c.RateGbps {
+				c.RateGbps = rate
+			}
+		}
+		out = append(out, c)
+	}
+	return out
+}
+
+// activeRate returns the fastest active port's rate on an adapter.
+//
+// Only ACTIVE ports carry traffic. A port that is physically linked but has no
+// subnet manager sits in INIT, looks healthy to anything counting adapters, and
+// passes no data — which is precisely the case worth catching, because it
+// survives a reboot and looks like a software problem.
+func activeRate(h HCA) (int, bool) {
+	best, ok := 0, false
+	for _, p := range h.Ports {
+		if !strings.EqualFold(p.State, "ACTIVE") {
+			continue
+		}
+		ok = true
+		if p.RateGbps > best {
+			best = p.RateGbps
+		}
+	}
+	return best, ok
+}
+
+func portStates(h HCA) string {
+	if len(h.Ports) == 0 {
+		return "no ports"
+	}
+	var parts []string
+	for _, p := range h.Ports {
+		parts = append(parts, fmt.Sprintf("port %d %s/%s", p.Num, p.State, p.PhysState))
+	}
+	return strings.Join(parts, ", ")
+}
+
+func listNames(hcas []HCA) string {
+	if len(hcas) == 0 {
+		return "none"
+	}
+	var names []string
+	for _, h := range hcas {
+		names = append(names, h.Name)
+	}
+	return strings.Join(names, ", ")
+}

internal/agent/discovery/fabrics_test.go

@@ -0,0 +1,114 @@
+package discovery
+
+import (
+	"strings"
+	"testing"
+)
+
+func hca(name string, ports ...Port) HCA { return HCA{Name: name, Ports: ports} }
+
+func active(rate int) Port {
+	return Port{Num: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: rate, LinkLayer: LinkInfiniBand}
+}
+
+func TestADeclaredAdapterThatIsNotThereIsReported(t *testing.T) {
+	// A typo, or a card that enumerated differently after a reboot. Taken on
+	// trust it becomes NCCL_IB_HCA naming an adapter that does not exist, and
+	// the replica hangs on its first all-reduce.
+	got := CheckFabrics(
+		map[string][]string{"ib-a": {"mlx5_0", "mlx5_9"}},
+		[]HCA{hca("mlx5_0", active(200))},
+	)
+	if len(got) != 1 {
+		t.Fatalf("got %d checks", len(got))
+	}
+	c := got[0]
+	if len(c.Usable) != 1 || c.Usable[0] != "mlx5_0" {
+		t.Errorf("usable = %v, want only the adapter that exists", c.Usable)
+	}
+	if len(c.Problems) != 1 || !strings.Contains(c.Problems[0], "mlx5_9") {
+		t.Errorf("problems = %v", c.Problems)
+	}
+	// The message names what is actually present, because the operator is
+	// looking at a typo and cannot see the node.
+	if !strings.Contains(c.Problems[0], "mlx5_0") {
+		t.Errorf("the problem does not say what was found: %q", c.Problems[0])
+	}
+}
+
+func TestAnAdapterWithNoActivePortCannotCarryTraffic(t *testing.T) {
+	// A port that is physically linked but has no subnet manager sits in INIT.
+	// It looks healthy to anything counting adapters and passes no data, which
+	// is the case worth catching: it survives a reboot and looks like software.
+	got := CheckFabrics(
+		map[string][]string{"ib-a": {"mlx5_0"}},
+		[]HCA{hca("mlx5_0", Port{Num: 1, State: "INIT", PhysState: "LinkUp", RateGbps: 200})},
+	)
+	c := got[0]
+	if c.OK() {
+		t.Error("a fabric with no active port was reported usable")
+	}
+	if len(c.Problems) != 1 || !strings.Contains(c.Problems[0], "INIT") {
+		t.Errorf("the problem does not say what state the port is in: %v", c.Problems)
+	}
+}
+
+func TestTheRateIsTheSlowestActiveLink(t *testing.T) {
+	// A collective runs at the speed of its slowest link, so reporting the
+	// fastest would have the scheduler believe a fabric is quicker than it is.
+	got := CheckFabrics(
+		map[string][]string{"ib-a": {"fast", "slow"}},
+		[]HCA{hca("fast", active(400)), hca("slow", active(100))},
+	)
+	if got[0].RateGbps != 100 {
+		t.Errorf("rate = %d, want 100", got[0].RateGbps)
+	}
+}
+
+func TestAnAdapterWithSeveralPortsUsesItsBestActiveOne(t *testing.T) {
+	// A dual-port card with one cable in is a perfectly good adapter, and
+	// judging it by the dead port would take a working fabric out of service.
+	got := CheckFabrics(
+		map[string][]string{"ib-a": {"mlx5_0"}},
+		[]HCA{hca("mlx5_0",
+			Port{Num: 1, State: "DOWN", PhysState: "Disabled"},
+			active(200),
+		)},
+	)
+	if !got[0].OK() {
+		t.Fatalf("a card with one live port was rejected: %+v", got[0])
+	}
+	if got[0].RateGbps != 200 {
+		t.Errorf("rate = %d, want the active port's 200", got[0].RateGbps)
+	}
+}
+
+func TestAFabricWithNothingUsableIsNotOK(t *testing.T) {
+	got := CheckFabrics(
+		map[string][]string{"ib-dead": {"mlx5_0"}},
+		[]HCA{hca("mlx5_0", Port{Num: 1, State: "DOWN", PhysState: "Polling"})},
+	)
+	if got[0].OK() {
+		t.Error("a fabric whose only adapter is down was reported usable")
+	}
+}
+
+func TestANodeWithNoFabricsDeclaredProducesNoChecks(t *testing.T) {
+	// Every node without InfiniBand, which must stay entirely unaffected.
+	if got := CheckFabrics(nil, []HCA{hca("mlx5_0", active(200))}); len(got) != 0 {
+		t.Errorf("got %d checks for a node that declared none", len(got))
+	}
+}
+
+func TestChecksComeBackInAStableOrder(t *testing.T) {
+	// Map iteration would otherwise reorder log lines between restarts, which
+	// makes a diff of two registrations unreadable.
+	declared := map[string][]string{"ib-c": {"m"}, "ib-a": {"m"}, "ib-b": {"m"}}
+	got := CheckFabrics(declared, nil)
+	if len(got) != 3 {
+		t.Fatalf("got %d", len(got))
+	}
+	if got[0].Fabric != "ib-a" || got[1].Fabric != "ib-b" || got[2].Fabric != "ib-c" {
+		t.Errorf("order = %s, %s, %s", got[0].Fabric, got[1].Fabric, got[2].Fabric)
+	}
+}

internal/agent/discovery/gpu.go

@@ -0,0 +1,150 @@
+package discovery
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	"regexp"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+// GPU is one accelerator on this node.
+type GPU struct {
+	Index int    `json:"index"`
+	UUID  string `json:"uuid"`
+	Name  string `json:"name"`
+	// MemoryTotalMB is the total framebuffer size.
+	MemoryTotalMB int `json:"memory_total_mb"`
+	// MemoryUsedMB is what is already allocated. Non-zero on an idle node means
+	// something other than our engines is using the card (plan §5.1).
+	MemoryUsedMB int `json:"memory_used_mb"`
+	// UtilizationPercent is how busy the card was over the last sample period.
+	//
+	// The number people actually watch, and the one nothing collected: memory
+	// used says a model is loaded, and this says whether it is doing anything.
+	// A fleet of cards at ninety percent occupancy and two percent utilisation
+	// is a fleet holding models nobody is asking for, which is invisible
+	// without it.
+	UtilizationPercent int `json:"utilization_percent,omitempty"`
+	// ComputeCapability gates quantization choices: an FP8 profile must not
+	// land on a card below 8.9 (plan §6.2).
+	ComputeCapability string `json:"compute_capability"`
+	NUMANode          int    `json:"numa_node"`
+	PCIBusID          string `json:"pci_bus_id,omitempty"`
+	DriverVersion     string `json:"driver_version,omitempty"`
+	CUDAVersion       string `json:"cuda_version,omitempty"`
+	// NVLinkPeers are the indices of GPUs reachable over NVLink. Within a node
+	// a TP group should stay inside one NVLink domain.
+	NVLinkPeers []int `json:"nvlink_peers,omitempty"`
+	// Health carries anything that should keep the GPU out of placement.
+	Health GPUHealth `json:"health"`
+}
+
+// GPUHealth records conditions that make a GPU unsafe to schedule onto.
+type GPUHealth struct {
+	// ECCErrors is the count of uncorrected ECC errors.
+	ECCErrors int `json:"ecc_errors"`
+	// TemperatureC is the current die temperature.
+	TemperatureC int `json:"temperature_c"`
+	// Problems lists human-readable reasons to avoid this GPU.
+	Problems []string `json:"problems,omitempty"`
+}
+
+// OK reports whether the GPU is fit for placement.
+func (h GPUHealth) OK() bool { return len(h.Problems) == 0 }
+
+// nonAlnum collapses everything that is not a letter or digit.
+var nonAlnum = regexp.MustCompile(`[^a-z0-9]+`)
+
+// Class derives the GPU class used for placement: same model, same memory.
+//
+// Two cards only belong in the same tensor-parallel group if they are the same
+// class, so the derivation has to be stable across driver versions and naming
+// quirks ("NVIDIA H100 80GB HBM3" and "H100-SXM5-80GB" are the same class).
+func (g GPU) Class() string {
+	name := strings.ToLower(g.Name)
+	name = strings.TrimPrefix(name, "nvidia ")
+	name = strings.TrimPrefix(name, "tesla ")
+
+	// Keep the model token: the first word that contains a digit, which is how
+	// NVIDIA names every accelerator (h100, a100, l40s, rtx4090).
+	var model string
+	for _, tok := range nonAlnum.Split(name, -1) {
+		if tok == "" || !strings.ContainsFunc(tok, func(r rune) bool { return r >= '0' && r <= '9' }) {
+			continue
+		}
+		// Skip memory and form-factor tokens, which are not the model.
+		if strings.HasSuffix(tok, "gb") || tok == "80gb" || tok == "sxm4" || tok == "sxm5" || tok == "pcie" {
+			continue
+		}
+		model = tok
+		break
+	}
+	if model == "" {
+		model = nonAlnum.ReplaceAllString(name, "-")
+	}
+
+	// Round to the nearest GiB: vendors report 81559 MiB for an "80GB" card.
+	gb := int(math.Round(float64(g.MemoryTotalMB) / 1024))
+	return fmt.Sprintf("%s-%dg", model, gb)
+}
+
+// MarshalJSON adds the derived class, so a consumer of the JSON does not have
+// to reimplement the derivation to know which GPUs are interchangeable.
+func (g GPU) MarshalJSON() ([]byte, error) {
+	type alias GPU // prevents recursing back into this method
+	return json.Marshal(struct {
+		alias
+		Class string `json:"class"`
+	}{alias(g), g.Class()})
+}
+
+// Inventory is everything discovery found on this node.
+type Inventory struct {
+	GPUs []GPU `json:"gpus"`
+	HCAs []HCA `json:"hcas"`
+	// Source names the backend that produced the GPU list, so a consumer can
+	// tell "no GPUs" from "discovery did not work".
+	Source string `json:"source"`
+	// MemoryTotalMB is the machine's RAM. Zero means unknown, which is what an
+	// agent older than this field reports, and is treated as no constraint.
+	MemoryTotalMB int `json:"memory_total_mb,omitempty"`
+	// Weights are the model files this node can see. Empty means it was not
+	// asked to look.
+	Weights []LocalWeights `json:"weights,omitempty"`
+	// Storage is the disk behind those weights, one entry per filesystem.
+	// Empty means the same thing it means for Weights: not asked.
+	Storage []Storage `json:"storage,omitempty"`
+}
+
+// Classes returns the distinct GPU classes present, sorted.
+func (inv Inventory) Classes() []string {
+	seen := map[string]struct{}{}
+	for _, g := range inv.GPUs {
+		seen[g.Class()] = struct{}{}
+	}
+	out := make([]string, 0, len(seen))
+	for c := range seen {
+		out = append(out, c)
+	}
+	slices.Sort(out)
+	return out
+}
+
+// parseCSVInt reads an integer from nvidia-smi's CSV output, tolerating the
+// "[N/A]" and "Not Supported" values it emits for unsupported queries.
+func parseCSVInt(s string) (int, bool) {
+	s = strings.TrimSpace(s)
+	s = strings.TrimSuffix(s, " MiB")
+	s = strings.TrimSuffix(s, " W")
+	if s == "" || strings.HasPrefix(s, "[") || strings.EqualFold(s, "Not Supported") {
+		return 0, false
+	}
+	v, err := strconv.Atoi(s)
+	if err != nil {
+		return 0, false
+	}
+	return v, true
+}

internal/agent/discovery/gpu_nonvml.go

@@ -0,0 +1,11 @@
+//go:build !nvml
+
+package discovery
+
+import "context"
+
+// nvmlAvailable reports that this build cannot use NVML.
+const nvmlAvailable = false
+
+// DiscoverGPUsNVML is unavailable in builds without the nvml tag.
+func DiscoverGPUsNVML(context.Context) ([]GPU, error) { return nil, ErrNoNVML }

internal/agent/discovery/gpu_nvml.go

@@ -0,0 +1,148 @@
+//go:build nvml
+
+// This file is the NVML-backed GPU discovery path, built only with the `nvml`
+// tag because the bindings require cgo. Without the tag the agent falls back to
+// nvidia-smi, which keeps the server side — and development machines without a
+// C toolchain or a driver — building cleanly (plan §3.2).
+//
+// What NVML adds over nvidia-smi is NVLink topology. Nothing else exposes which
+// GPUs share an NVLink domain, and that is what keeps a tensor-parallel group
+// off the slow path inside a node (plan §6.2, constraint 3).
+
+package discovery
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/NVIDIA/go-nvml/pkg/nvml"
+)
+
+// nvmlAvailable reports that this build can use NVML.
+const nvmlAvailable = true
+
+// DiscoverGPUsNVML enumerates GPUs through NVML.
+//
+// It returns an error rather than falling back on its own: the caller decides
+// whether to degrade to nvidia-smi, because on a node that is supposed to have
+// working NVML a silent downgrade hides a broken driver installation.
+func DiscoverGPUsNVML(_ context.Context) ([]GPU, error) {
+	if ret := nvml.Init(); ret != nvml.SUCCESS {
+		return nil, fmt.Errorf("nvml init: %s", nvml.ErrorString(ret))
+	}
+	defer func() { _ = nvml.Shutdown() }()
+
+	driver, _ := nvml.SystemGetDriverVersion()
+	cuda := ""
+	if v, ret := nvml.SystemGetCudaDriverVersion(); ret == nvml.SUCCESS {
+		// NVML reports 12040 for 12.4.
+		cuda = fmt.Sprintf("%d.%d", v/1000, (v%1000)/10)
+	}
+
+	count, ret := nvml.DeviceGetCount()
+	if ret != nvml.SUCCESS {
+		return nil, fmt.Errorf("nvml device count: %s", nvml.ErrorString(ret))
+	}
+
+	// Bus id to index, so NVLink peers can be reported as indices. NVML gives
+	// the remote end of a link as a PCI address, which is not what placement
+	// works in.
+	byBusID := make(map[string]int, count)
+	gpus := make([]GPU, 0, count)
+
+	for i := range count {
+		dev, ret := nvml.DeviceGetHandleByIndex(i)
+		if ret != nvml.SUCCESS {
+			return nil, fmt.Errorf("nvml device %d: %s", i, nvml.ErrorString(ret))
+		}
+
+		g := GPU{Index: i, NUMANode: -1, DriverVersion: driver, CUDAVersion: cuda}
+		g.UUID, _ = dev.GetUUID()
+		g.Name, _ = dev.GetName()
+
+		if mem, ret := dev.GetMemoryInfo(); ret == nvml.SUCCESS {
+			g.MemoryTotalMB = int(mem.Total / (1 << 20))
+			g.MemoryUsedMB = int(mem.Used / (1 << 20))
+		}
+		if major, minor, ret := dev.GetCudaComputeCapability(); ret == nvml.SUCCESS {
+			g.ComputeCapability = fmt.Sprintf("%d.%d", major, minor)
+		}
+		if pci, ret := dev.GetPciInfo(); ret == nvml.SUCCESS {
+			g.PCIBusID = busIDString(pci.BusId[:])
+			byBusID[normalizeBusID(g.PCIBusID)] = i
+		}
+		if t, ret := dev.GetTemperature(nvml.TEMPERATURE_GPU); ret == nvml.SUCCESS {
+			g.Health.TemperatureC = int(t)
+		}
+		// Volatile counters reset on driver reload, which is what we want: an
+		// error from three months and one reboot ago is not a reason to refuse
+		// a card today.
+		if n, ret := dev.GetTotalEccErrors(nvml.MEMORY_ERROR_TYPE_UNCORRECTED, nvml.VOLATILE_ECC); ret == nvml.SUCCESS {
+			g.Health.ECCErrors = int(n)
+		}
+
+		assessHealth(&g)
+		gpus = append(gpus, g)
+	}
+
+	// A second pass, because a link's remote end may be a GPU enumerated later.
+	for i := range gpus {
+		dev, ret := nvml.DeviceGetHandleByIndex(gpus[i].Index)
+		if ret != nvml.SUCCESS {
+			continue
+		}
+		gpus[i].NVLinkPeers = nvlinkPeers(dev, byBusID, gpus[i].Index)
+	}
+	return gpus, nil
+}
+
+// nvlinkPeers returns the indices of GPUs reachable from dev over NVLink.
+func nvlinkPeers(dev nvml.Device, byBusID map[string]int, self int) []int {
+	seen := map[int]struct{}{}
+	var out []int
+
+	for link := range nvml.NVLINK_MAX_LINKS {
+		pci, ret := dev.GetNvLinkRemotePciInfo(link)
+		if ret != nvml.SUCCESS {
+			// Unpopulated links return an error; that is the normal case for
+			// most of the 36 possible links, so it is not worth reporting.
+			continue
+		}
+		peer, ok := byBusID[normalizeBusID(busIDString(pci.BusId[:]))]
+		if !ok || peer == self {
+			// The remote end can be an NVSwitch rather than a GPU, in which
+			// case it is not in the map.
+			continue
+		}
+		if _, dup := seen[peer]; dup {
+			continue
+		}
+		seen[peer] = struct{}{}
+		out = append(out, peer)
+	}
+	return out
+}
+
+// busIDString turns NVML's fixed-size C char array into a Go string.
+func busIDString(b []int8) string {
+	buf := make([]byte, 0, len(b))
+	for _, c := range b {
+		if c == 0 {
+			break
+		}
+		buf = append(buf, byte(c))
+	}
+	return string(buf)
+}
+
+// normalizeBusID makes bus ids comparable: NVML reports them uppercase from one
+// call and lowercase from another, and sometimes with a shortened domain.
+func normalizeBusID(s string) string {
+	s = strings.ToLower(strings.TrimSpace(s))
+	// "0000:1b:00.0" and "00000000:1B:00.0" are the same device.
+	if i := strings.Index(s, ":"); i > 0 {
+		s = strings.TrimLeft(s[:i], "0") + s[i:]
+	}
+	return strings.TrimPrefix(s, ":")
+}

internal/agent/discovery/gpu_smi.go

@@ -0,0 +1,126 @@
+package discovery
+
+import (
+	"context"
+	"encoding/csv"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+)
+
+// smiQueryFields are the columns requested from nvidia-smi, in order.
+var smiQueryFields = []string{
+	"index",
+	"uuid",
+	"name",
+	"memory.total",
+	"memory.used",
+	"compute_cap",
+	"pci.bus_id",
+	"driver_version",
+	"temperature.gpu",
+	"ecc.errors.uncorrected.volatile.total",
+	// Last, and optional. An nvidia-smi too old to know this field returns a
+	// shorter row, and every column before it still lines up — which is why it
+	// goes on the end rather than anywhere else, and why the parser requires
+	// only the mandatory prefix.
+	"utilization.gpu",
+}
+
+// mandatorySMIFields is how many columns a row must have to be usable.
+//
+// Everything before utilization.gpu. A driver that does not know the last field
+// gives a short row, and rejecting those would make an llmgw upgrade silently
+// stop discovering GPUs on an older node — a far worse outcome than not knowing
+// how busy a card is.
+const mandatorySMIFields = 10
+
+// DiscoverGPUsSMI enumerates GPUs by shelling out to nvidia-smi.
+//
+// This is the cgo-free path (plan §3.2): the NVML bindings need cgo, which
+// would force the whole binary — server included — to be built with a C
+// toolchain. nvidia-smi gives everything except NVLink topology, which the
+// NVML build adds.
+func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) {
+	if binary == "" {
+		// Not configured and not installed is a node without an NVIDIA card,
+		// not a failed discovery. Reported as an error it also cut discovery
+		// short before InfiniBand was looked at, on every CPU node that had
+		// not been told by hand to skip GPUs.
+		if _, err := exec.LookPath("nvidia-smi"); err != nil {
+			return nil, nil
+		}
+		binary = "nvidia-smi"
+	}
+	out, err := exec.CommandContext(ctx, binary,
+		"--query-gpu="+strings.Join(smiQueryFields, ","),
+		"--format=csv,noheader,nounits",
+	).Output()
+	if err != nil {
+		return nil, fmt.Errorf("running %s: %w", binary, err)
+	}
+	return parseSMI(string(out))
+}
+
+func parseSMI(out string) ([]GPU, error) {
+	r := csv.NewReader(strings.NewReader(out))
+	r.TrimLeadingSpace = true
+	r.FieldsPerRecord = -1 // tolerate driver versions with extra columns
+
+	rows, err := r.ReadAll()
+	if err != nil {
+		return nil, fmt.Errorf("parsing nvidia-smi output: %w", err)
+	}
+
+	gpus := make([]GPU, 0, len(rows))
+	for _, row := range rows {
+		if len(row) < mandatorySMIFields {
+			continue
+		}
+		idx, err := strconv.Atoi(strings.TrimSpace(row[0]))
+		if err != nil {
+			continue
+		}
+
+		g := GPU{
+			Index:             idx,
+			UUID:              strings.TrimSpace(row[1]),
+			Name:              strings.TrimSpace(row[2]),
+			ComputeCapability: strings.TrimSpace(row[5]),
+			PCIBusID:          strings.TrimSpace(row[6]),
+			DriverVersion:     strings.TrimSpace(row[7]),
+			NUMANode:          -1,
+		}
+		g.MemoryTotalMB, _ = parseCSVInt(row[3])
+		g.MemoryUsedMB, _ = parseCSVInt(row[4])
+		g.Health.TemperatureC, _ = parseCSVInt(row[8])
+		g.Health.ECCErrors, _ = parseCSVInt(row[9])
+		if len(row) > 10 {
+			g.UtilizationPercent, _ = parseCSVInt(row[10])
+		}
+
+		assessHealth(&g)
+		gpus = append(gpus, g)
+	}
+	return gpus, nil
+}
+
+// residentMemoryToleranceMB is how much allocated memory is written off as
+// driver overhead rather than a foreign process.
+const residentMemoryToleranceMB = 512
+
+// assessHealth flags conditions that should keep a GPU out of placement.
+func assessHealth(g *GPU) {
+	if g.Health.ECCErrors > 0 {
+		g.Health.Problems = append(g.Health.Problems,
+			fmt.Sprintf("%d uncorrected ECC errors", g.Health.ECCErrors))
+	}
+	// Memory already in use means something else is on the card. Starting an
+	// engine there would either fail to allocate or quietly evict the other
+	// process's throughput.
+	if g.MemoryUsedMB > residentMemoryToleranceMB {
+		g.Health.Problems = append(g.Health.Problems,
+			fmt.Sprintf("%d MiB already allocated by another process", g.MemoryUsedMB))
+	}
+}

internal/agent/discovery/gpu_test.go

@@ -0,0 +1,179 @@
+package discovery
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+const smiSample = `0, GPU-11111111-2222-3333-4444-555555555555, NVIDIA H100 80GB HBM3, 81559, 4, 9.0, 00000000:1B:00.0, 550.54.15, 34, 0
+1, GPU-66666666-7777-8888-9999-000000000000, NVIDIA H100 80GB HBM3, 81559, 0, 9.0, 00000000:1C:00.0, 550.54.15, 36, 0
+`
+
+func TestParseSMI(t *testing.T) {
+	gpus, err := parseSMI(smiSample)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(gpus) != 2 {
+		t.Fatalf("expected 2 GPUs, got %d", len(gpus))
+	}
+
+	g := gpus[0]
+	if g.Index != 0 || g.UUID != "GPU-11111111-2222-3333-4444-555555555555" {
+		t.Errorf("identity wrong: %+v", g)
+	}
+	if g.MemoryTotalMB != 81559 {
+		t.Errorf("MemoryTotalMB=%d", g.MemoryTotalMB)
+	}
+	if g.ComputeCapability != "9.0" {
+		t.Errorf("ComputeCapability=%q; FP8 placement depends on it", g.ComputeCapability)
+	}
+	if g.DriverVersion != "550.54.15" {
+		t.Errorf("DriverVersion=%q", g.DriverVersion)
+	}
+	if !g.Health.OK() {
+		t.Errorf("a healthy GPU was flagged: %v", g.Health.Problems)
+	}
+}
+
+func TestParseSMIUnsupportedFields(t *testing.T) {
+	// Consumer cards report [N/A] for ECC; that must not fail the whole parse.
+	const row = `0, GPU-abc, NVIDIA GeForce RTX 4090, 24564, 0, 8.9, 00000000:01:00.0, 550.54.15, 45, [N/A]
+`
+	gpus, err := parseSMI(row)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(gpus) != 1 {
+		t.Fatalf("expected 1 GPU, got %d", len(gpus))
+	}
+	if gpus[0].Health.ECCErrors != 0 || !gpus[0].Health.OK() {
+		t.Errorf("[N/A] ECC was misread: %+v", gpus[0].Health)
+	}
+}
+
+func TestHealthFlagsForeignMemory(t *testing.T) {
+	// Memory in use on an otherwise idle card means another process is there.
+	const row = `0, GPU-abc, NVIDIA H100 80GB HBM3, 81559, 40000, 9.0, 00000000:1B:00.0, 550.54.15, 34, 0
+`
+	gpus, _ := parseSMI(row)
+	if gpus[0].Health.OK() {
+		t.Error("40 GiB allocated by another process was not flagged")
+	}
+}
+
+func TestHealthFlagsECC(t *testing.T) {
+	const row = `0, GPU-abc, NVIDIA H100 80GB HBM3, 81559, 0, 9.0, 00000000:1B:00.0, 550.54.15, 34, 3
+`
+	gpus, _ := parseSMI(row)
+	if gpus[0].Health.OK() {
+		t.Error("uncorrected ECC errors were not flagged")
+	}
+}
+
+func TestGPUClass(t *testing.T) {
+	// The same card must derive the same class however the driver names it,
+	// because a tensor-parallel group may not mix classes.
+	cases := []struct {
+		name   string
+		memMB  int
+		expect string
+	}{
+		{"NVIDIA H100 80GB HBM3", 81559, "h100-80g"},
+		{"NVIDIA H100-SXM5-80GB", 81559, "h100-80g"},
+		{"Tesla A100-SXM4-40GB", 40960, "a100-40g"},
+		{"NVIDIA A100 80GB PCIe", 81920, "a100-80g"},
+		// Sizes come from what the driver reports, in GiB. That does not always
+		// match the marketing name: an L40S is sold as 48GB but reports 44.99
+		// GiB, because the marketing figure is decimal GB. Being consistent
+		// matters more than matching the box, since the number is what decides
+		// whether a model fits.
+		{"NVIDIA L40S", 46068, "l40s-45g"},
+		// The model token is what distinguishes a 4090 from a 4080, so "rtx"
+		// is not enough on its own.
+		{"NVIDIA GeForce RTX 4090", 24564, "4090-24g"},
+	}
+	for _, c := range cases {
+		got := GPU{Name: c.name, MemoryTotalMB: c.memMB}.Class()
+		if got != c.expect {
+			t.Errorf("%q (%d MiB) -> %q, want %q", c.name, c.memMB, got, c.expect)
+		}
+	}
+}
+
+func TestInventoryClasses(t *testing.T) {
+	inv := Inventory{GPUs: []GPU{
+		{Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559},
+		{Name: "NVIDIA H100-SXM5-80GB", MemoryTotalMB: 81559},
+		{Name: "Tesla A100-SXM4-40GB", MemoryTotalMB: 40960},
+	}}
+	got := inv.Classes()
+	if len(got) != 2 || got[0] != "a100-40g" || got[1] != "h100-80g" {
+		t.Errorf("classes=%v", got)
+	}
+}
+
+// TestDiscoverGPUsSMI runs the real code path against a stub binary, so the
+// command construction is covered too, not just the parser.
+func TestDiscoverGPUsSMI(t *testing.T) {
+	dir := t.TempDir()
+	stub := filepath.Join(dir, "nvidia-smi")
+	script := "#!/bin/sh\ncat <<'EOF'\n" + smiSample + "EOF\n"
+	if err := os.WriteFile(stub, []byte(script), 0o755); err != nil {
+		t.Fatal(err)
+	}
+
+	gpus, err := DiscoverGPUsSMI(t.Context(), stub)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(gpus) != 2 {
+		t.Fatalf("expected 2 GPUs, got %d", len(gpus))
+	}
+}
+
+func TestDiscoverGPUsSMIMissingBinary(t *testing.T) {
+	// A node with no NVIDIA driver must produce a clear error rather than a
+	// panic or an empty success.
+	_, err := DiscoverGPUsSMI(context.Background(), filepath.Join(t.TempDir(), "absent"))
+	if err == nil {
+		t.Error("expected an error when nvidia-smi is missing")
+	}
+}
+
+func TestAnOlderDriverWithoutUtilisationStillDiscovers(t *testing.T) {
+	// An nvidia-smi too old for utilization.gpu returns a shorter row. Rejecting
+	// those would make an llmgw upgrade silently stop discovering GPUs on an
+	// older node, which is far worse than not knowing how busy a card is — and
+	// is what the first version of this change did.
+	old := "0, GPU-aaa, NVIDIA H100 80GB HBM3, 81559, 1024, 9.0, 0000:01:00.0, 550.54, 41, 0"
+	gpus, err := parseSMI(old)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(gpus) != 1 {
+		t.Fatalf("a row without utilisation produced %d GPUs", len(gpus))
+	}
+	if gpus[0].MemoryTotalMB != 81559 || gpus[0].Health.TemperatureC != 41 {
+		t.Errorf("columns shifted: %+v", gpus[0])
+	}
+	if gpus[0].UtilizationPercent != 0 {
+		t.Errorf("utilisation = %d, want 0 when unreported", gpus[0].UtilizationPercent)
+	}
+}
+
+func TestUtilisationIsReadWhenPresent(t *testing.T) {
+	row := "0, GPU-aaa, NVIDIA H100 80GB HBM3, 81559, 1024, 9.0, 0000:01:00.0, 550.54, 41, 0, 87"
+	gpus, err := parseSMI(row)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(gpus) != 1 {
+		t.Fatalf("got %d GPUs", len(gpus))
+	}
+	if gpus[0].UtilizationPercent != 87 {
+		t.Errorf("utilisation = %d, want 87", gpus[0].UtilizationPercent)
+	}
+}

internal/agent/discovery/host.go

@@ -0,0 +1,154 @@
+package discovery
+
+import (
+	"bufio"
+	"os"
+	"runtime"
+	"strings"
+)
+
+// Host is what a node is, as far as installing software on it is concerned.
+type Host struct {
+	// OSID, OSVersion and OSCodename come from /etc/os-release: "ubuntu",
+	// "24.04", "noble".
+	OSID       string `json:"os_id,omitempty"`
+	OSVersion  string `json:"os_version,omitempty"`
+	OSCodename string `json:"os_codename,omitempty"`
+	// OSLike is ID_LIKE: the families this system says it follows, "debian"
+	// for Ubuntu, "rhel centos fedora" for Rocky. It is what decides the
+	// package manager, which the name alone does not for every derivative.
+	OSLike []string `json:"os_like,omitempty"`
+	// Arch is Debian's spelling, "amd64" or "arm64", because that is what
+	// package names and release archives are keyed by.
+	Arch string `json:"arch,omitempty"`
+	// CPUFlags are the vector extensions inference cares about, of those this
+	// processor has.
+	CPUFlags []string `json:"cpu_flags,omitempty"`
+	// NVIDIADriver is the loaded driver's version, empty when there is none.
+	NVIDIADriver string `json:"nvidia_driver,omitempty"`
+}
+
+// Family is the packaging family: "deb", "rpm", or "" when neither is known.
+func (h Host) Family() string {
+	for _, id := range append([]string{h.OSID}, h.OSLike...) {
+		switch id {
+		case "debian", "ubuntu":
+			return "deb"
+		case "rhel", "fedora", "centos", "rocky", "almalinux", "suse", "opensuse":
+			return "rpm"
+		}
+	}
+	return ""
+}
+
+// HasCPUFlag reports whether the processor has an extension.
+func (h Host) HasCPUFlag(f string) bool {
+	for _, have := range h.CPUFlags {
+		if have == f {
+			return true
+		}
+	}
+	return false
+}
+
+// interestingFlags are the extensions that decide which build of an engine
+// runs well. A VM on QEMU's default CPU model has none of them — and still
+// runs, five times slower, with nothing saying why.
+var interestingFlags = []string{"avx", "avx2", "fma", "f16c", "avx512f", "avx512_bf16", "amx_tile", "asimd", "sve"}
+
+// DiscoverHost reads what the machine is.
+func DiscoverHost() Host {
+	h := hostFromOSRelease("/etc/os-release")
+	h.Arch = debianArch(runtime.GOARCH)
+	h.CPUFlags = cpuFlagsFrom("/proc/cpuinfo")
+	h.NVIDIADriver = nvidiaDriverFrom("/proc/driver/nvidia/version")
+	return h
+}
+
+func hostFromOSRelease(path string) Host {
+	var h Host
+	f, err := os.Open(path) //nolint:gosec // a fixed system path
+	if err != nil {
+		return h
+	}
+	defer f.Close()
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		k, v, ok := strings.Cut(sc.Text(), "=")
+		if !ok {
+			continue
+		}
+		v = strings.Trim(v, `"'`)
+		switch k {
+		case "ID":
+			h.OSID = v
+		case "VERSION_ID":
+			h.OSVersion = v
+		case "VERSION_CODENAME":
+			h.OSCodename = v
+		case "ID_LIKE":
+			h.OSLike = strings.Fields(v)
+		}
+	}
+	return h
+}
+
+func debianArch(goarch string) string {
+	switch goarch {
+	case "amd64", "arm64":
+		return goarch
+	case "386":
+		return "i386"
+	case "arm":
+		return "armhf"
+	}
+	return goarch
+}
+
+func cpuFlagsFrom(path string) []string {
+	b, err := os.ReadFile(path) //nolint:gosec // a fixed system path
+	if err != nil {
+		return nil
+	}
+	have := map[string]bool{}
+	for _, line := range strings.Split(string(b), "\n") {
+		k, v, ok := strings.Cut(line, ":")
+		if !ok {
+			continue
+		}
+		// "flags" on x86, "Features" on ARM. The first processor is enough:
+		// a machine does not mix instruction sets across its cores.
+		if k = strings.TrimSpace(k); k != "flags" && k != "Features" {
+			continue
+		}
+		for _, f := range strings.Fields(v) {
+			have[f] = true
+		}
+		break
+	}
+	var out []string
+	for _, f := range interestingFlags {
+		if have[f] {
+			out = append(out, f)
+		}
+	}
+	return out
+}
+
+// nvidiaDriverFrom reads the loaded kernel module's version, which exists
+// exactly when a driver is loaded — unlike nvidia-smi, which can be installed
+// on a machine whose module failed to load.
+func nvidiaDriverFrom(path string) string {
+	b, err := os.ReadFile(path) //nolint:gosec // a fixed system path
+	if err != nil {
+		return ""
+	}
+	// "NVRM version: NVIDIA UNIX x86_64 Kernel Module  570.86.15  Thu ..."
+	fields := strings.Fields(string(b))
+	for i, f := range fields {
+		if f == "Module" && i+1 < len(fields) {
+			return fields[i+1]
+		}
+	}
+	return ""
+}

internal/agent/discovery/ib.go

@@ -0,0 +1,203 @@
+// Package discovery inspects a node's GPUs, InfiniBand adapters and the
+// topology between them (plan §5.1 and §5.2).
+package discovery
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+// SysfsRoot is the default place InfiniBand devices appear.
+const SysfsRoot = "/sys/class/infiniband"
+
+// LinkLayer distinguishes real InfiniBand from RoCE.
+type LinkLayer string
+
+const (
+	// LinkInfiniBand is native InfiniBand.
+	LinkInfiniBand LinkLayer = "InfiniBand"
+	// LinkEthernet is RoCE — Ethernet carrying RDMA.
+	LinkEthernet LinkLayer = "Ethernet"
+)
+
+// Port is one port of an HCA.
+type Port struct {
+	Num int `json:"num"`
+	// State is the port's logical state; only ACTIVE ports carry traffic.
+	State string `json:"state"`
+	// PhysState is the physical link state, useful for telling "cable unplugged"
+	// from "no subnet manager".
+	PhysState string `json:"phys_state"`
+	// RateGbps is the negotiated rate.
+	RateGbps  int       `json:"rate_gbps"`
+	LinkLayer LinkLayer `json:"link_layer"`
+	// GIDPrefix is the subnet prefix from gids/0, normalized to lowercase hex
+	// groups. It is the only automatic fabric hint available, and an unreliable
+	// one: OpenSM's default is fe80:: everywhere (plan §5.2).
+	GIDPrefix string `json:"gid_prefix"`
+	LID       int    `json:"lid"`
+	SMLID     int    `json:"sm_lid"`
+}
+
+// Active reports whether the port can carry traffic.
+func (p Port) Active() bool { return strings.EqualFold(p.State, "ACTIVE") }
+
+// HCA is one host channel adapter.
+type HCA struct {
+	Name     string `json:"name"`
+	Ports    []Port `json:"ports"`
+	PCIAddr  string `json:"pci_addr,omitempty"`
+	NUMANode int    `json:"numa_node"`
+	// FirmwareVersion is reported by the driver; worth capturing because
+	// NCCL behaviour differs between firmware revisions.
+	FirmwareVersion string `json:"firmware_version,omitempty"`
+}
+
+// ActivePorts returns the ports that can carry traffic.
+func (h HCA) ActivePorts() []Port {
+	var out []Port
+	for _, p := range h.Ports {
+		if p.Active() {
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
+// DiscoverIB enumerates the InfiniBand adapters under root.
+//
+// A missing root is not an error: plenty of nodes have no IB at all, and the
+// scheduler treats them as single-node islands (plan §2).
+func DiscoverIB(root string) ([]HCA, error) {
+	if root == "" {
+		root = SysfsRoot
+	}
+	entries, err := os.ReadDir(root)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("reading %s: %w", root, err)
+	}
+
+	var out []HCA
+	for _, e := range entries {
+		hca, err := readHCA(filepath.Join(root, e.Name()), e.Name())
+		if err != nil {
+			// One unreadable adapter must not hide the rest; the node may still
+			// be usable through its other HCAs.
+			continue
+		}
+		out = append(out, hca)
+	}
+	slices.SortFunc(out, func(a, b HCA) int { return strings.Compare(a.Name, b.Name) })
+	return out, nil
+}
+
+func readHCA(dir, name string) (HCA, error) {
+	h := HCA{Name: name, NUMANode: -1}
+	h.FirmwareVersion = readTrimmed(filepath.Join(dir, "fw_ver"))
+
+	// device is a symlink into the PCI tree; its target's base name is the
+	// bus address, and numa_node lives beside it.
+	if target, err := filepath.EvalSymlinks(filepath.Join(dir, "device")); err == nil {
+		h.PCIAddr = filepath.Base(target)
+		if n, err := strconv.Atoi(readTrimmed(filepath.Join(target, "numa_node"))); err == nil {
+			h.NUMANode = n
+		}
+	}
+
+	portDirs, err := os.ReadDir(filepath.Join(dir, "ports"))
+	if err != nil {
+		return h, fmt.Errorf("%s: no ports: %w", name, err)
+	}
+	for _, pd := range portDirs {
+		num, err := strconv.Atoi(pd.Name())
+		if err != nil {
+			continue
+		}
+		h.Ports = append(h.Ports, readPort(filepath.Join(dir, "ports", pd.Name()), num))
+	}
+	slices.SortFunc(h.Ports, func(a, b Port) int { return a.Num - b.Num })
+	return h, nil
+}
+
+func readPort(dir string, num int) Port {
+	p := Port{
+		Num:       num,
+		State:     parseEnumeratedState(readTrimmed(filepath.Join(dir, "state"))),
+		PhysState: parseEnumeratedState(readTrimmed(filepath.Join(dir, "phys_state"))),
+		RateGbps:  parseRate(readTrimmed(filepath.Join(dir, "rate"))),
+		LinkLayer: LinkLayer(readTrimmed(filepath.Join(dir, "link_layer"))),
+		GIDPrefix: parseGIDPrefix(readTrimmed(filepath.Join(dir, "gids", "0"))),
+		LID:       parseMaybeHex(readTrimmed(filepath.Join(dir, "lid"))),
+		SMLID:     parseMaybeHex(readTrimmed(filepath.Join(dir, "sm_lid"))),
+	}
+	return p
+}
+
+// parseEnumeratedState turns sysfs's "4: ACTIVE" into "ACTIVE".
+func parseEnumeratedState(s string) string {
+	if _, rest, found := strings.Cut(s, ":"); found {
+		return strings.TrimSpace(rest)
+	}
+	return s
+}
+
+// parseRate turns "100 Gb/sec (4X EDR)" into 100.
+func parseRate(s string) int {
+	f := strings.Fields(s)
+	if len(f) == 0 {
+		return 0
+	}
+	// The rate can be fractional ("2.5 Gb/sec") on old hardware; truncate.
+	v, err := strconv.ParseFloat(f[0], 64)
+	if err != nil {
+		return 0
+	}
+	return int(v)
+}
+
+// parseGIDPrefix keeps the first four groups of a GID, which is the subnet
+// prefix; the remaining four are the interface id.
+func parseGIDPrefix(gid string) string {
+	parts := strings.Split(strings.ToLower(gid), ":")
+	if len(parts) < 4 {
+		return ""
+	}
+	return strings.Join(parts[:4], ":")
+}
+
+// parseMaybeHex accepts both "0x1a" and "26"; sysfs is inconsistent between
+// drivers and kernel versions.
+func parseMaybeHex(s string) int {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return 0
+	}
+	if after, ok := strings.CutPrefix(strings.ToLower(s), "0x"); ok {
+		v, err := strconv.ParseInt(after, 16, 64)
+		if err != nil {
+			return 0
+		}
+		return int(v)
+	}
+	v, err := strconv.Atoi(s)
+	if err != nil {
+		return 0
+	}
+	return v
+}
+
+func readTrimmed(path string) string {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(b))
+}

internal/agent/discovery/ib_test.go

@@ -0,0 +1,216 @@
+package discovery
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// fakeSysfs builds an InfiniBand tree shaped like the real one.
+func fakeSysfs(t *testing.T) string {
+	t.Helper()
+	root := t.TempDir()
+
+	write := func(path, content string) {
+		t.Helper()
+		full := filepath.Join(root, path)
+		if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(full, []byte(content+"\n"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// mlx5_0: one active IB port.
+	write("ib/mlx5_0/fw_ver", "20.35.1012")
+	write("ib/mlx5_0/ports/1/state", "4: ACTIVE")
+	write("ib/mlx5_0/ports/1/phys_state", "5: LinkUp")
+	write("ib/mlx5_0/ports/1/rate", "200 Gb/sec (4X HDR)")
+	write("ib/mlx5_0/ports/1/link_layer", "InfiniBand")
+	write("ib/mlx5_0/ports/1/gids/0", "FEC0:0000:0000:000A:0002:C903:00A0:1234")
+	write("ib/mlx5_0/ports/1/lid", "0x1a")
+	write("ib/mlx5_0/ports/1/sm_lid", "1")
+
+	// mlx5_1: two ports, the second one down.
+	write("ib/mlx5_1/ports/1/state", "4: ACTIVE")
+	write("ib/mlx5_1/ports/1/rate", "200 Gb/sec (4X HDR)")
+	write("ib/mlx5_1/ports/1/link_layer", "InfiniBand")
+	write("ib/mlx5_1/ports/1/gids/0", "fec0:0000:0000:000a:0002:c903:00a0:5678")
+	write("ib/mlx5_1/ports/2/state", "1: DOWN")
+	write("ib/mlx5_1/ports/2/phys_state", "3: Disabled")
+	write("ib/mlx5_1/ports/2/rate", "0")
+	write("ib/mlx5_1/ports/2/link_layer", "InfiniBand")
+
+	// mlx5_2: RoCE rather than native IB.
+	write("ib/mlx5_2/ports/1/state", "4: ACTIVE")
+	write("ib/mlx5_2/ports/1/rate", "100 Gb/sec (4X EDR)")
+	write("ib/mlx5_2/ports/1/link_layer", "Ethernet")
+	write("ib/mlx5_2/ports/1/gids/0", "0000:0000:0000:0000:0000:ffff:0a00:0001")
+
+	// A PCI device with a NUMA node, linked the way the driver does it.
+	pci := filepath.Join(root, "pci", "0000:0b:00.0")
+	if err := os.MkdirAll(pci, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(pci, "numa_node"), []byte("1\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Symlink(pci, filepath.Join(root, "ib", "mlx5_0", "device")); err != nil {
+		t.Fatal(err)
+	}
+
+	return filepath.Join(root, "ib")
+}
+
+func TestDiscoverIB(t *testing.T) {
+	hcas, err := DiscoverIB(fakeSysfs(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(hcas) != 3 {
+		t.Fatalf("expected 3 HCAs, got %d", len(hcas))
+	}
+	// Sorted by name, so the order is predictable for the rest of the test.
+	if hcas[0].Name != "mlx5_0" || hcas[2].Name != "mlx5_2" {
+		t.Fatalf("HCAs are not sorted by name: %v %v", hcas[0].Name, hcas[2].Name)
+	}
+
+	h := hcas[0]
+	if h.FirmwareVersion != "20.35.1012" {
+		t.Errorf("fw_ver=%q", h.FirmwareVersion)
+	}
+	if h.PCIAddr != "0000:0b:00.0" {
+		t.Errorf("PCIAddr=%q", h.PCIAddr)
+	}
+	if h.NUMANode != 1 {
+		t.Errorf("NUMANode=%d; GPU-to-HCA affinity depends on this", h.NUMANode)
+	}
+
+	p := h.Ports[0]
+	if p.State != "ACTIVE" || !p.Active() {
+		t.Errorf("state=%q", p.State)
+	}
+	if p.PhysState != "LinkUp" {
+		t.Errorf("phys_state=%q", p.PhysState)
+	}
+	if p.RateGbps != 200 {
+		t.Errorf("rate=%d", p.RateGbps)
+	}
+	if p.LinkLayer != LinkInfiniBand {
+		t.Errorf("link_layer=%q", p.LinkLayer)
+	}
+	if p.GIDPrefix != "fec0:0000:0000:000a" {
+		t.Errorf("GIDPrefix=%q; it must be lowercased and cut to four groups", p.GIDPrefix)
+	}
+	if p.LID != 26 {
+		t.Errorf("LID=%d, expected 0x1a to parse as 26", p.LID)
+	}
+	if p.SMLID != 1 {
+		t.Errorf("SMLID=%d, expected a decimal value to parse too", p.SMLID)
+	}
+}
+
+func TestActivePortsExcludesDown(t *testing.T) {
+	hcas, err := DiscoverIB(fakeSysfs(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	h := hcas[1] // mlx5_1
+	if len(h.Ports) != 2 {
+		t.Fatalf("expected 2 ports, got %d", len(h.Ports))
+	}
+	active := h.ActivePorts()
+	if len(active) != 1 || active[0].Num != 1 {
+		t.Errorf("a DOWN port was reported as active: %+v", active)
+	}
+}
+
+func TestRoCEIsDistinguished(t *testing.T) {
+	// RoCE is usable but behaves differently from native IB; the placement
+	// logic needs to be able to tell them apart.
+	hcas, err := DiscoverIB(fakeSysfs(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := hcas[2].Ports[0].LinkLayer; got != LinkEthernet {
+		t.Errorf("mlx5_2 link_layer=%q, expected Ethernet (RoCE)", got)
+	}
+}
+
+func TestMissingRootIsNotAnError(t *testing.T) {
+	// A node without InfiniBand is perfectly normal; it just forms its own
+	// single-node island.
+	hcas, err := DiscoverIB(filepath.Join(t.TempDir(), "nonexistent"))
+	if err != nil {
+		t.Fatalf("a node without IB should not be an error: %v", err)
+	}
+	if len(hcas) != 0 {
+		t.Errorf("expected no HCAs, got %d", len(hcas))
+	}
+}
+
+func TestUnreadableAdapterDoesNotHideOthers(t *testing.T) {
+	root := fakeSysfs(t)
+	// An adapter with no ports directory at all.
+	if err := os.MkdirAll(filepath.Join(root, "mlx5_broken"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+
+	hcas, err := DiscoverIB(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(hcas) != 3 {
+		t.Errorf("a broken adapter changed the result: got %d HCAs", len(hcas))
+	}
+}
+
+func TestParseHelpers(t *testing.T) {
+	t.Run("rate", func(t *testing.T) {
+		for in, want := range map[string]int{
+			"200 Gb/sec (4X HDR)": 200,
+			"100 Gb/sec (4X EDR)": 100,
+			"2.5 Gb/sec (1X SDR)": 2,
+			"":                    0,
+			"garbage":             0,
+		} {
+			if got := parseRate(in); got != want {
+				t.Errorf("parseRate(%q)=%d, want %d", in, got, want)
+			}
+		}
+	})
+
+	t.Run("state", func(t *testing.T) {
+		for in, want := range map[string]string{
+			"4: ACTIVE": "ACTIVE",
+			"1: DOWN":   "DOWN",
+			"5: LinkUp": "LinkUp",
+			"ACTIVE":    "ACTIVE",
+			"":          "",
+		} {
+			if got := parseEnumeratedState(in); got != want {
+				t.Errorf("parseEnumeratedState(%q)=%q, want %q", in, got, want)
+			}
+		}
+	})
+
+	t.Run("hex", func(t *testing.T) {
+		for in, want := range map[string]int{
+			"0x1a": 26, "0X1A": 26, "26": 26, "": 0, "zzz": 0,
+		} {
+			if got := parseMaybeHex(in); got != want {
+				t.Errorf("parseMaybeHex(%q)=%d, want %d", in, got, want)
+			}
+		}
+	})
+
+	t.Run("gid", func(t *testing.T) {
+		if got := parseGIDPrefix("FE80:0000:0000:0000:0002:C903:00A0:1234"); got != "fe80:0000:0000:0000" {
+			t.Errorf("parseGIDPrefix=%q", got)
+		}
+		if got := parseGIDPrefix("short:gid"); got != "" {
+			t.Errorf("a malformed GID should yield an empty prefix, got %q", got)
+		}
+	})
+}

internal/agent/discovery/json_test.go

@@ -0,0 +1,40 @@
+package discovery
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestJSONShape(t *testing.T) {
+	inv := Inventory{
+		Source: "nvidia-smi",
+		GPUs:   []GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, ComputeCapability: "9.0"}},
+	}
+	b, err := json.Marshal(inv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var out struct {
+		Source string `json:"source"`
+		GPUs   []struct {
+			Index int    `json:"index"`
+			Class string `json:"class"`
+			Mem   int    `json:"memory_total_mb"`
+		} `json:"gpus"`
+	}
+	if err := json.Unmarshal(b, &out); err != nil {
+		t.Fatal(err)
+	}
+	if out.Source != "nvidia-smi" || len(out.GPUs) != 1 {
+		t.Fatalf("shape wrong: %s", b)
+	}
+	// The derived class is what decides which GPUs are interchangeable, so a
+	// consumer must not have to recompute it.
+	if out.GPUs[0].Class != "h100-80g" {
+		t.Errorf("class=%q, should be included in the JSON", out.GPUs[0].Class)
+	}
+	if out.GPUs[0].Mem != 81559 {
+		t.Errorf("memory_total_mb=%d", out.GPUs[0].Mem)
+	}
+}

internal/agent/discovery/memory.go

@@ -0,0 +1,51 @@
+package discovery
+
+import (
+	"bufio"
+	"os"
+	"strconv"
+	"strings"
+)
+
+// MemoryTotalMB reports the machine's total RAM.
+//
+// It exists because placement had no idea how much a node could hold. A CPU
+// profile has no card to be free, so every ready node looked like a candidate:
+// a 14B model whose working set is 24 GB would have been placed just as
+// happily on a node with 8 GB, which reports itself perfectly healthy right up
+// until the OOM killer arrives. Labels were the stopgap; this is the fact they
+// were standing in for.
+//
+// Read from /proc/meminfo, which is Linux-only — every node is. On anything
+// else it returns 0, which callers treat as "unknown" and therefore no
+// constraint, exactly as an older agent that does not report it at all.
+func MemoryTotalMB() int { return memTotalMBFrom("/proc/meminfo") }
+
+func memTotalMBFrom(path string) int {
+	f, err := os.Open(path)
+	if err != nil {
+		return 0
+	}
+	defer f.Close()
+
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		rest, ok := strings.CutPrefix(sc.Text(), "MemTotal:")
+		if !ok {
+			continue
+		}
+		fields := strings.Fields(rest)
+		if len(fields) == 0 {
+			return 0
+		}
+		kb, err := strconv.Atoi(fields[0])
+		if err != nil {
+			return 0
+		}
+		// MemTotal is what the kernel can hand out, already short of the
+		// installed total by the firmware's reservations. That is the right
+		// number for deciding what fits.
+		return kb / 1024
+	}
+	return 0
+}

internal/agent/discovery/memory_test.go

@@ -0,0 +1,39 @@
+package discovery
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestMemTotalIsReadFromMeminfo(t *testing.T) {
+	dir := t.TempDir()
+	p := filepath.Join(dir, "meminfo")
+	// The real shape, including the fields before and after, so the parser is
+	// not tested against something tidier than it will see.
+	if err := os.WriteFile(p, []byte(
+		"MemTotal:       32876244 kB\nMemFree:         1234 kB\nMemAvailable:   5678 kB\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if got, want := memTotalMBFrom(p), 32876244/1024; got != want {
+		t.Errorf("memTotalMBFrom = %d, want %d", got, want)
+	}
+}
+
+func TestUnreadableMeminfoReportsUnknownRatherThanZeroMemory(t *testing.T) {
+	// Zero means "did not say" everywhere downstream, and placement treats it
+	// as no constraint. A node with genuinely no memory is not a case.
+	if got := memTotalMBFrom(filepath.Join(t.TempDir(), "absent")); got != 0 {
+		t.Errorf("got %d for a missing file", got)
+	}
+	dir := t.TempDir()
+	p := filepath.Join(dir, "meminfo")
+	for _, junk := range []string{"", "MemTotal:\n", "MemTotal: not-a-number kB\n", "Other: 1 kB\n"} {
+		if err := os.WriteFile(p, []byte(junk), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		if got := memTotalMBFrom(p); got != 0 {
+			t.Errorf("got %d for %q", got, junk)
+		}
+	}
+}

internal/agent/discovery/storage.go

@@ -0,0 +1,90 @@
+package discovery
+
+import (
+	"os"
+	"syscall"
+)
+
+// Storage is one filesystem a node keeps weights on.
+//
+// One entry per filesystem rather than per configured directory, because free
+// space belongs to the disk and not to the path. Two model directories on the
+// same disk are one pool of bytes, and reporting them separately would let the
+// control plane add 24 GB to 24 GB and place a 40 GB model on a node with 24.
+type Storage struct {
+	// Path is one directory on this filesystem — the first one configured that
+	// landed here, so the operator recognises it.
+	Path string `json:"path"`
+	// FreeBytes is what a non-root process can still write. Deliberately not
+	// the root reserve: the agent does not run as root, so the reserve is space
+	// it cannot have.
+	FreeBytes int64 `json:"free_bytes"`
+	// TotalBytes is the filesystem's size, so a reading of "8 GB free" can be
+	// told apart on a 10 TB filer and on a 10 GB root partition.
+	TotalBytes int64 `json:"total_bytes"`
+	// Shared marks a filesystem the node did not have to fill. Free space on
+	// one is shared with every other node using it, so two nodes reporting
+	// 500 GB free on the same filer have 500 GB between them, not a terabyte.
+	Shared bool `json:"shared,omitempty"`
+}
+
+// ScanStorage reports the filesystems behind the given model directories.
+//
+// Empty in, empty out, and a directory that cannot be read is skipped rather
+// than reported as full: a mount that has not come back yet looks exactly like
+// a missing directory, and claiming zero free bytes would take the node out of
+// placement for a reason that is not true. Absent is not zero here, as
+// everywhere else in this system.
+func ScanStorage(dirs []string) []Storage {
+	var out []Storage
+	seen := map[uint64]int{} // filesystem id -> index in out
+
+	for _, dir := range dirs {
+		if dir == "" {
+			continue
+		}
+		dev, ok := filesystemID(dir)
+		if !ok {
+			continue
+		}
+		if i, dup := seen[dev]; dup {
+			// Same disk, already counted. Keep the first path: it is the one
+			// the operator wrote first and so the one they will recognise.
+			_ = i
+			continue
+		}
+
+		var st syscall.Statfs_t
+		if err := syscall.Statfs(dir, &st); err != nil {
+			continue
+		}
+		// Bsize is int32 on some platforms and int64 on others, which is why
+		// every field here is widened explicitly rather than assumed.
+		block := int64(st.Bsize)
+		seen[dev] = len(out)
+		out = append(out, Storage{
+			Path:       dir,
+			FreeBytes:  int64(st.Bavail) * block, //nolint:gosec,unconvert // widened on purpose
+			TotalBytes: int64(st.Blocks) * block, //nolint:gosec,unconvert // widened on purpose
+			Shared:     isNetworkFS(dir),
+		})
+	}
+	return out
+}
+
+// filesystemID identifies the disk a path is on.
+//
+// By device number rather than by the statfs filesystem id, because the field
+// holding that id is spelled differently on Linux and on macOS and this has to
+// compile on both — the agent runs on one and is developed on the other.
+func filesystemID(path string) (uint64, bool) {
+	fi, err := os.Stat(path)
+	if err != nil {
+		return 0, false
+	}
+	st, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0, false
+	}
+	return uint64(st.Dev), true //nolint:gosec,unconvert // int32 on macOS, uint64 on Linux
+}

internal/agent/discovery/storage_test.go

@@ -0,0 +1,63 @@
+package discovery
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestTwoDirectoriesOnOneDiskAreOnePoolOfBytes(t *testing.T) {
+	// The reason this matters: free space belongs to the disk, not to the path.
+	// Counting two directories separately lets the control plane add 24 GB to
+	// 24 GB and place a 40 GB model on a node that has 24.
+	base := t.TempDir()
+	a := filepath.Join(base, "models-a")
+	b := filepath.Join(base, "models-b")
+	for _, d := range []string{a, b} {
+		if err := os.Mkdir(d, 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	got := ScanStorage([]string{a, b})
+	if len(got) != 1 {
+		t.Fatalf("two directories on one filesystem reported as %d pools: %+v", len(got), got)
+	}
+	if got[0].Path != a {
+		t.Errorf("kept %q rather than the first directory configured", got[0].Path)
+	}
+}
+
+func TestStorageReportsSomethingPlausible(t *testing.T) {
+	got := ScanStorage([]string{t.TempDir()})
+	if len(got) != 1 {
+		t.Fatalf("a real directory reported %d filesystems", len(got))
+	}
+	s := got[0]
+	if s.TotalBytes <= 0 {
+		t.Error("a filesystem with no size at all")
+	}
+	if s.FreeBytes < 0 || s.FreeBytes > s.TotalBytes {
+		t.Errorf("free %d of total %d", s.FreeBytes, s.TotalBytes)
+	}
+}
+
+func TestAMissingDirectoryIsNotAFullDisk(t *testing.T) {
+	// A mount that has not come back yet looks exactly like a missing
+	// directory. Reporting it as zero free bytes would take the node out of
+	// placement for a reason that is not true; reporting nothing says only
+	// that nothing is known, which is the honest answer.
+	got := ScanStorage([]string{filepath.Join(t.TempDir(), "never-mounted")})
+	if len(got) != 0 {
+		t.Errorf("a missing directory was reported as a filesystem: %+v", got)
+	}
+}
+
+func TestNoDirectoriesConstrainNothing(t *testing.T) {
+	if got := ScanStorage(nil); len(got) != 0 {
+		t.Errorf("scanning nothing found %+v", got)
+	}
+	if got := ScanStorage([]string{""}); len(got) != 0 {
+		t.Errorf("an empty path found %+v", got)
+	}
+}

internal/agent/discovery/weights.go

@@ -0,0 +1,137 @@
+package discovery
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"syscall"
+)
+
+// LocalWeights is one model file or directory a node can see.
+type LocalWeights struct {
+	Path      string `json:"path"`
+	SizeBytes int64  `json:"size_bytes"`
+	// Shared marks weights on a filesystem this node did not have to fill.
+	//
+	// Worth distinguishing: twenty nodes each holding their own copy and twenty
+	// nodes seeing one filer are the same list of paths and completely
+	// different situations — one has twenty copies of a 141 GB model and the
+	// other has one.
+	Shared bool `json:"shared,omitempty"`
+}
+
+// ScanWeights lists the model files under the given directories.
+//
+// Storage is the part of this system that will differ at every installation —
+// a Lustre mount, an image exported over the fabric, local NVMe, an rsync from
+// a laptop — so llmgw does not try to own it. It observes the result instead:
+// whatever put the file there, the question placement needs answered is only
+// whether it is present.
+//
+// Scanning nothing is the default and constrains nothing. A node with no
+// directories configured behaves exactly as every node did before this existed,
+// which is what makes the whole feature optional.
+func ScanWeights(dirs []string) []LocalWeights {
+	var out []LocalWeights
+	seen := map[string]struct{}{}
+
+	for _, dir := range dirs {
+		if dir == "" {
+			continue
+		}
+		shared := isNetworkFS(dir)
+		entries, err := os.ReadDir(dir)
+		if err != nil {
+			// A directory that is not there is not an error to report upward:
+			// a mount that has not come back yet looks exactly like this, and
+			// the node is simply one without those weights until it does.
+			continue
+		}
+		for _, e := range entries {
+			path := filepath.Join(dir, e.Name())
+			if _, dup := seen[path]; dup {
+				continue
+			}
+			size, ok := weightSize(path, e)
+			if !ok {
+				continue
+			}
+			seen[path] = struct{}{}
+			out = append(out, LocalWeights{Path: path, SizeBytes: size, Shared: shared})
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path })
+	return out
+}
+
+// weightSize measures a candidate and says whether it is one at all.
+//
+// A model is either a single file — a .gguf, usually — or a directory of
+// shards and a config, which is what every HuggingFace checkout looks like.
+// Anything else in the directory is somebody's notes and is not reported.
+func weightSize(path string, e fs.DirEntry) (int64, bool) {
+	if e.IsDir() {
+		// A directory counts only if it actually holds weights. An empty one
+		// is a half-finished download, and reporting it as present would have
+		// placement choose a node whose engine then fails to load.
+		var total int64
+		var found bool
+		_ = filepath.WalkDir(path, func(p string, d fs.DirEntry, err error) error {
+			if err != nil || d.IsDir() {
+				return nil //nolint:nilerr // an unreadable subtree is not fatal
+			}
+			if isWeightFile(p) {
+				found = true
+			}
+			if info, err := d.Info(); err == nil {
+				total += info.Size()
+			}
+			return nil
+		})
+		return total, found
+	}
+	if !isWeightFile(path) {
+		return 0, false
+	}
+	info, err := e.Info()
+	if err != nil {
+		return 0, false
+	}
+	return info.Size(), true
+}
+
+func isWeightFile(path string) bool {
+	switch strings.ToLower(filepath.Ext(path)) {
+	case ".gguf", ".safetensors", ".bin", ".pt", ".pth":
+		return true
+	}
+	return false
+}
+
+// isNetworkFS reports whether a path is on a filesystem the node did not fill.
+//
+// By magic number rather than by parsing mounts, because the mount table says
+// what was asked for and this says what the kernel actually has. The list is
+// the network filesystems that turn up in front of GPUs: NFS, Lustre, GPFS,
+// CephFS, GlusterFS and SMB. Anything unrecognised is treated as local, which
+// is the conservative direction — a shared filer mistaken for local storage
+// costs an unnecessary copy, while local storage mistaken for shared would have
+// the cluster believe a model is everywhere when it is on one node.
+func isNetworkFS(path string) bool {
+	var st syscall.Statfs_t
+	if err := syscall.Statfs(path, &st); err != nil {
+		return false
+	}
+	switch uint32(st.Type) { //nolint:unconvert // Type is int64 on some platforms
+	case 0x6969, // NFS
+		0x0BD00BD0, // Lustre
+		0x47504653, // GPFS
+		0x00C36400, // CephFS
+		0x65735546, // FUSE, which is how GlusterFS and many object mounts appear
+		0xFF534D42: // SMB/CIFS
+		return true
+	}
+	return false
+}

internal/agent/discovery/weights_test.go

@@ -0,0 +1,114 @@
+package discovery
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func write(t *testing.T, path string, size int) {
+	t.Helper()
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, make([]byte, size), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestScanFindsFilesAndDirectoriesOfWeights(t *testing.T) {
+	dir := t.TempDir()
+	write(t, filepath.Join(dir, "qwen-3b-q4.gguf"), 1024)
+	// A HuggingFace checkout: a directory of shards rather than one file.
+	write(t, filepath.Join(dir, "Llama-3.3-70B", "model-00001.safetensors"), 2048)
+	write(t, filepath.Join(dir, "Llama-3.3-70B", "config.json"), 64)
+
+	got := ScanWeights([]string{dir})
+	if len(got) != 2 {
+		t.Fatalf("found %d entries: %+v", len(got), got)
+	}
+	byName := map[string]int64{}
+	for _, w := range got {
+		byName[filepath.Base(w.Path)] = w.SizeBytes
+	}
+	if byName["qwen-3b-q4.gguf"] != 1024 {
+		t.Errorf("gguf size = %d", byName["qwen-3b-q4.gguf"])
+	}
+	// The directory's size is everything in it, config included: what matters
+	// is how much would have to be moved to put it somewhere else.
+	if byName["Llama-3.3-70B"] != 2048+64 {
+		t.Errorf("checkout size = %d, want %d", byName["Llama-3.3-70B"], 2048+64)
+	}
+}
+
+func TestAnEmptyDirectoryIsNotWeights(t *testing.T) {
+	// A half-finished download looks exactly like this, and reporting it as
+	// present would have placement choose a node whose engine then fails to
+	// load — the failure this whole feature exists to avoid.
+	dir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(dir, "downloading"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	write(t, filepath.Join(dir, "downloading", "README.md"), 10)
+
+	if got := ScanWeights([]string{dir}); len(got) != 0 {
+		t.Errorf("a directory with no weight files was reported: %+v", got)
+	}
+}
+
+func TestNotesAndScriptsAreNotReportedAsModels(t *testing.T) {
+	dir := t.TempDir()
+	write(t, filepath.Join(dir, "notes.txt"), 10)
+	write(t, filepath.Join(dir, "fetch.sh"), 10)
+	write(t, filepath.Join(dir, "real.gguf"), 10)
+
+	got := ScanWeights([]string{dir})
+	if len(got) != 1 || filepath.Base(got[0].Path) != "real.gguf" {
+		t.Errorf("scan returned %+v", got)
+	}
+}
+
+func TestAMissingDirectoryIsNotAnError(t *testing.T) {
+	// A mount that has not come back yet looks exactly like a directory that is
+	// not there, and the node is simply one without those weights until it is.
+	got := ScanWeights([]string{filepath.Join(t.TempDir(), "never-existed")})
+	if len(got) != 0 {
+		t.Errorf("got %+v", got)
+	}
+}
+
+func TestScanningNothingIsTheDefaultAndConstrainsNothing(t *testing.T) {
+	// What makes the feature optional: an agent with no directories configured
+	// behaves exactly as every agent did before this existed.
+	if got := ScanWeights(nil); len(got) != 0 {
+		t.Errorf("got %+v", got)
+	}
+	if got := ScanWeights([]string{""}); len(got) != 0 {
+		t.Errorf("an empty directory name produced %+v", got)
+	}
+}
+
+func TestTheSamePathIsReportedOnce(t *testing.T) {
+	// Two configured directories can overlap, or be the same mount reached two
+	// ways. Counting a model twice would overstate what a fabric holds.
+	dir := t.TempDir()
+	write(t, filepath.Join(dir, "m.gguf"), 10)
+	if got := ScanWeights([]string{dir, dir}); len(got) != 1 {
+		t.Errorf("the same directory twice produced %+v", got)
+	}
+}
+
+func TestLocalStorageIsNeverMistakenForShared(t *testing.T) {
+	// The conservative direction. A shared filer read as local costs an
+	// unnecessary copy; local read as shared would have the cluster believe a
+	// model is everywhere when it is on one node.
+	dir := t.TempDir()
+	write(t, filepath.Join(dir, "m.gguf"), 10)
+	got := ScanWeights([]string{dir})
+	if len(got) != 1 {
+		t.Fatalf("got %+v", got)
+	}
+	if got[0].Shared {
+		t.Error("a temporary directory on local disk was reported as shared")
+	}
+}

internal/agent/engines/engine.go

@@ -0,0 +1,338 @@
+// Package engines builds the command lines and environments that start
+// inference engines (plan §5.3).
+//
+// Nothing here starts a process; that is the supervisor's job. Keeping command
+// construction pure makes it testable without GPUs, which matters because the
+// environment generation is where multi-node runs go wrong: a single missing
+// NCCL variable turns a working tensor-parallel group into one that hangs on
+// the first all-reduce.
+package engines
+
+import (
+	"fmt"
+	"os"
+	"slices"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// Profile is how a model runs on a GPU class. It mirrors the catalog profile
+// (plan §6.1), reduced to what launching needs.
+type Profile struct {
+	Name     string
+	Engine   string
+	GPUClass string
+
+	// TP and PP are tensor and pipeline parallel degrees.
+	TP int
+	PP int
+	// GPUsPerNode is how many GPUs each shard occupies.
+	GPUsPerNode int
+	// Nodes is how many nodes the replica spans. Above 1, a fabric is required.
+	Nodes int
+
+	// Args are passed through to the engine untouched.
+	Args []string
+	// Env overrides anything the engine adapter generates.
+	Env map[string]string
+
+	ModelPath string
+	// ServedModelName is the name clients use, which need not match the path.
+	ServedModelName string
+}
+
+// ShardSpec is one shard of one replica on this node.
+type ShardSpec struct {
+	ReplicaID string
+	ShardID   string
+	// Rank is the node rank; 0 is the leader and serves HTTP.
+	Rank   int
+	NNodes int
+
+	// LeaderAddr and LeaderPort are the distributed init endpoint, always on the
+	// management network rather than the fabric: NCCL bootstraps over TCP and
+	// only then moves to RDMA.
+	LeaderAddr string
+	LeaderPort int
+
+	// GPUIndices are the local GPUs this shard uses.
+	GPUIndices []int
+	// HCAs are the adapters of the chosen fabric, and only that fabric. A shard
+	// must never be given HCAs from two fabrics.
+	HCAs []string
+	// MgmtInterface is the ethernet interface used for bootstrap traffic.
+	MgmtInterface string
+
+	HTTPPort int
+	Profile  Profile
+
+	// Pool and Fabric are control plane concepts the agent cannot derive. They
+	// are carried so the agent can report them back after a restart, and are
+	// not used when building a command.
+	Pool   string
+	Fabric string
+	// ProfileFP identifies the version of the profile this shard was started
+	// from. Carried for the same reason as Pool and Fabric, and never read: it
+	// lets the control plane tell, after its own restart, whether a shard that
+	// is still running matches the profile the catalog holds now.
+	ProfileFP string
+}
+
+// Leader reports whether this shard serves HTTP for the replica.
+func (s ShardSpec) Leader() bool { return s.Rank == 0 }
+
+// Command is a fully resolved launch: what to exec, with which environment.
+type Command struct {
+	Path string
+	Args []string
+	// Env is the complete environment, not a delta.
+	Env []string
+	// Name distinguishes the processes of a multi-process shard in logs.
+	Name string
+	// WaitForLeaderPort makes the supervisor hold the next command until
+	// LeaderAddr:LeaderPort accepts a connection.
+	//
+	// It exists for Ray: the head has to be listening before a worker can join,
+	// and both sides express the same wait because both dial the same address.
+	WaitForLeaderPort bool
+}
+
+// Engine builds launch commands for one inference engine.
+type Engine interface {
+	// Name is the engine's identifier in the catalog.
+	Name() string
+	// Validate rejects a profile that cannot work on these GPUs, before
+	// anything is started.
+	Validate(p Profile, gpus []GPUInfo) error
+
+	// Commands returns the processes that make up one shard, in start order.
+	//
+	// Most shards are a single process. vLLM under Ray is not: the leader runs
+	// a Ray head and then the server against it, and a worker runs only the Ray
+	// join. Returning a sequence rather than one command is what lets the
+	// supervisor treat both the same way — every command but the last is a
+	// sidecar kept alive for the shard's lifetime, and the last one's exit is
+	// the shard's exit.
+	Commands(s ShardSpec) ([]Command, error)
+
+	// ServesHTTP reports whether this rank answers the OpenAI API.
+	//
+	// In a multi-node replica only the leader does. Health-checking a worker's
+	// port would time out and fail a replica that is coming up perfectly well,
+	// which is exactly what happened before this existed.
+	ServesHTTP(s ShardSpec) bool
+
+	// HealthPath and MetricsPath are appended to the shard's base URL.
+	HealthPath() string
+	MetricsPath() string
+}
+
+// GPUInfo is the subset of GPU facts validation needs.
+type GPUInfo struct {
+	Index             int
+	Name              string
+	ComputeCapability string
+	MemoryTotalMB     int
+}
+
+// Registry holds the known engines.
+type Registry struct {
+	engines map[string]Engine
+}
+
+// NewRegistry returns a registry with the built-in engines.
+func NewRegistry(cfg Config) *Registry {
+	r := &Registry{engines: map[string]Engine{}}
+	r.Add(&VLLM{cfg: cfg.VLLM})
+	r.Add(&SGLang{cfg: cfg.SGLang})
+	r.Add(&LlamaCpp{cfg: cfg.LlamaCpp})
+	r.Add(&Sim{cfg: cfg.Sim})
+	return r
+}
+
+// UseInstalled points an engine at a build the agent installed, and reports
+// whether the engine takes one.
+func (r *Registry) UseInstalled(name, path string) bool {
+	u, ok := r.engines[name].(interface{ UseInstalled(string) })
+	if ok {
+		u.UseInstalled(path)
+	}
+	return ok
+}
+
+// Add registers an engine, replacing any engine of the same name.
+func (r *Registry) Add(e Engine) { r.engines[e.Name()] = e }
+
+// Get returns an engine by name.
+func (r *Registry) Get(name string) (Engine, error) {
+	e, ok := r.engines[name]
+	if !ok {
+		return nil, fmt.Errorf("unknown engine %q (have %s)", name, strings.Join(r.Names(), ", "))
+	}
+	return e, nil
+}
+
+// Names lists the registered engines.
+func (r *Registry) Names() []string {
+	out := make([]string, 0, len(r.engines))
+	for n := range r.engines {
+		out = append(out, n)
+	}
+	slices.Sort(out)
+	return out
+}
+
+// Config configures the engine adapters.
+type Config struct {
+	VLLM     VLLMConfig     `koanf:"vllm"`
+	SGLang   SGLangConfig   `koanf:"sglang"`
+	LlamaCpp LlamaCppConfig `koanf:"llamacpp"`
+	Sim      SimConfig      `koanf:"sim"`
+}
+
+// --- shared helpers ---
+
+// baseEnv builds the environment every engine shares.
+//
+// This is the part of §5.3 the plan marks critical. The rules encoded here:
+//
+//   - CUDA_VISIBLE_DEVICES pins the shard to its GPUs, so two shards on one
+//     node cannot fight over the same card.
+//   - NCCL_IB_HCA lists only the chosen fabric's adapters. Leaving it unset
+//     lets NCCL pick adapters from another fabric, where the peer is
+//     unreachable, and the job hangs instead of failing.
+//   - NCCL_SOCKET_IFNAME pins bootstrap to the management interface. NCCL
+//     otherwise probes every interface, including IB-over-IP addresses that
+//     cannot route between fabrics.
+//   - NCCL_IB_DISABLE is set when there is no fabric, so NCCL stops looking for
+//     one instead of timing out.
+func baseEnv(s ShardSpec) map[string]string {
+	env := map[string]string{
+		"CUDA_VISIBLE_DEVICES": joinInts(s.GPUIndices),
+	}
+
+	if len(s.HCAs) > 0 {
+		env["NCCL_IB_HCA"] = strings.Join(s.HCAs, ",")
+		env["NCCL_IB_DISABLE"] = "0"
+	} else {
+		// Single node, or a node with no fabric. Either way IB is not the path.
+		env["NCCL_IB_DISABLE"] = "1"
+	}
+
+	if s.MgmtInterface != "" {
+		env["NCCL_SOCKET_IFNAME"] = s.MgmtInterface
+		env["GLOO_SOCKET_IFNAME"] = s.MgmtInterface
+	}
+
+	// Profile overrides win: they are the escape hatch for the per-cluster
+	// tuning no adapter can guess.
+	for k, v := range s.Profile.Env {
+		env[k] = v
+	}
+	return env
+}
+
+// materializeEnv merges generated variables over the current process
+// environment and returns it in exec form.
+//
+// Inheriting matters: engines need PATH, HOME, HF_HOME, LD_LIBRARY_PATH and
+// whatever else the operator set up in the unit file.
+func materializeEnv(generated map[string]string) []string {
+	merged := map[string]string{}
+	for _, kv := range os.Environ() {
+		if k, v, ok := strings.Cut(kv, "="); ok {
+			merged[k] = v
+		}
+	}
+	for k, v := range generated {
+		merged[k] = v
+	}
+
+	out := make([]string, 0, len(merged))
+	for k, v := range merged {
+		out = append(out, k+"="+v)
+	}
+	// Sorted so a command is reproducible and diffable in logs.
+	sort.Strings(out)
+	return out
+}
+
+func joinInts(v []int) string {
+	parts := make([]string, len(v))
+	for i, n := range v {
+		parts[i] = strconv.Itoa(n)
+	}
+	return strings.Join(parts, ",")
+}
+
+// validateCommon holds the checks that apply to every engine.
+func validateCommon(p Profile, gpus []GPUInfo) error {
+	if p.ModelPath == "" {
+		return fmt.Errorf("profile %q: model path is empty", p.Name)
+	}
+	if p.TP <= 0 {
+		return fmt.Errorf("profile %q: tensor parallel degree must be positive", p.Name)
+	}
+	if p.Nodes > 1 && p.GPUsPerNode <= 0 {
+		return fmt.Errorf("profile %q: gpus_per_node is required for a %d-node profile", p.Name, p.Nodes)
+	}
+
+	perNode := p.GPUsPerNode
+	if perNode == 0 {
+		perNode = p.TP
+	}
+	nodes := max(p.Nodes, 1)
+	if want := perNode * nodes; want != p.TP*max(p.PP, 1) {
+		return fmt.Errorf("profile %q: %d nodes x %d GPUs does not match TP %d x PP %d",
+			p.Name, nodes, perNode, p.TP, max(p.PP, 1))
+	}
+	if len(gpus) > 0 && len(gpus) < perNode {
+		return fmt.Errorf("profile %q: needs %d GPUs per node, node has %d", p.Name, perNode, len(gpus))
+	}
+
+	// A tensor-parallel group must be uniform: mixing classes makes the slowest
+	// card set the pace for every collective, and mismatched memory makes the
+	// smallest card decide what fits.
+	if len(gpus) > 1 {
+		first := gpus[0].Name
+		for _, g := range gpus[1:] {
+			if g.Name != first {
+				return fmt.Errorf("profile %q: GPUs are not uniform (%q and %q); "+
+					"a tensor-parallel group may not mix GPU types", p.Name, first, g.Name)
+			}
+		}
+	}
+	return nil
+}
+
+// requireComputeCapability rejects profiles whose quantization the hardware
+// cannot execute.
+func requireComputeCapability(p Profile, gpus []GPUInfo, minCap float64, why string) error {
+	for _, g := range gpus {
+		cap, err := strconv.ParseFloat(g.ComputeCapability, 64)
+		if err != nil {
+			// Unknown capability: let it through rather than block a launch on
+			// a parsing problem. The engine will refuse it clearly enough.
+			continue
+		}
+		if cap < minCap {
+			return fmt.Errorf("profile %q: %s needs compute capability %.1f, GPU %d (%s) has %.1f",
+				p.Name, why, minCap, g.Index, g.Name, cap)
+		}
+	}
+	return nil
+}
+
+// quantizationOf extracts the --quantization argument, if present.
+func quantizationOf(args []string) string {
+	for i, a := range args {
+		if a == "--quantization" && i+1 < len(args) {
+			return args[i+1]
+		}
+		if v, ok := strings.CutPrefix(a, "--quantization="); ok {
+			return v
+		}
+	}
+	return ""
+}

internal/agent/engines/engines_test.go

@@ -0,0 +1,425 @@
+package engines
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+	"testing"
+)
+
+// buildOne returns the single command a shard produces, failing if it produced
+// a different number.
+func buildOne(t *testing.T, e Engine, s ShardSpec) Command {
+	t.Helper()
+	cmds := build(t, e, s)
+	if len(cmds) != 1 {
+		t.Fatalf("expected 1 command, got %d: %s", len(cmds), summarize(cmds))
+	}
+	return cmds[0]
+}
+
+// buildEngine returns the last command, which is the engine itself. The ones
+// before it are sidecars.
+func buildEngine(t *testing.T, e Engine, s ShardSpec) Command {
+	t.Helper()
+	cmds := build(t, e, s)
+	return cmds[len(cmds)-1]
+}
+
+func build(t *testing.T, e Engine, s ShardSpec) []Command {
+	t.Helper()
+	cmds, err := e.Commands(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(cmds) == 0 {
+		t.Fatal("no commands produced")
+	}
+	return cmds
+}
+
+// summarize keeps failure output readable: a Command carries the whole
+// environment, and printing it buries the actual problem.
+func summarize(cmds []Command) string {
+	var b strings.Builder
+	for i, c := range cmds {
+		if i > 0 {
+			b.WriteString(" | ")
+		}
+		fmt.Fprintf(&b, "%s: %s %s", c.Name, c.Path, strings.Join(c.Args, " "))
+	}
+	return b.String()
+}
+
+// envMap turns exec-form environment back into a map for assertions.
+func envMap(env []string) map[string]string {
+	m := map[string]string{}
+	for _, kv := range env {
+		if k, v, ok := strings.Cut(kv, "="); ok {
+			m[k] = v
+		}
+	}
+	return m
+}
+
+// argValue returns the value following flag, or "" if the flag is absent.
+func argValue(args []string, flag string) string {
+	i := slices.Index(args, flag)
+	if i < 0 || i+1 >= len(args) {
+		return ""
+	}
+	return args[i+1]
+}
+
+func singleNodeSpec() ShardSpec {
+	return ShardSpec{
+		ReplicaID:     "r1",
+		ShardID:       "r1-0",
+		Rank:          0,
+		NNodes:        1,
+		GPUIndices:    []int{0, 1, 2, 3},
+		MgmtInterface: "eth0",
+		HTTPPort:      20000,
+		Profile: Profile{
+			Name:      "h100-tp4",
+			Engine:    "vllm",
+			TP:        4,
+			ModelPath: "/models/llama",
+		},
+	}
+}
+
+func multiNodeSpec(rank int) ShardSpec {
+	s := singleNodeSpec()
+	s.Rank = rank
+	s.NNodes = 2
+	s.GPUIndices = []int{0, 1, 2, 3, 4, 5, 6, 7}
+	s.HCAs = []string{"mlx5_0", "mlx5_1"}
+	s.LeaderAddr = "10.0.0.10"
+	s.LeaderPort = 20001
+	s.Profile.TP = 16
+	s.Profile.GPUsPerNode = 8
+	s.Profile.Nodes = 2
+	return s
+}
+
+// --- environment generation: the part that breaks multi-node runs ---
+
+func TestSingleNodeDisablesIB(t *testing.T) {
+	cmd := buildOne(t, &VLLM{}, singleNodeSpec())
+	env := envMap(cmd.Env)
+
+	if env["NCCL_IB_DISABLE"] != "1" {
+		t.Error("NCCL_IB_DISABLE must be 1 without a fabric, or NCCL waits for one that is not there")
+	}
+	if _, set := env["NCCL_IB_HCA"]; set {
+		t.Error("NCCL_IB_HCA was set without any HCAs")
+	}
+	if env["CUDA_VISIBLE_DEVICES"] != "0,1,2,3" {
+		t.Errorf("CUDA_VISIBLE_DEVICES=%q", env["CUDA_VISIBLE_DEVICES"])
+	}
+}
+
+func TestMultiNodePinsFabric(t *testing.T) {
+	cmd := buildEngine(t, &VLLM{}, multiNodeSpec(0))
+	env := envMap(cmd.Env)
+
+	// Only the chosen fabric's adapters. Leaving this open lets NCCL pick an
+	// adapter on another fabric, where the peer is unreachable, and the job
+	// hangs rather than failing.
+	if env["NCCL_IB_HCA"] != "mlx5_0,mlx5_1" {
+		t.Errorf("NCCL_IB_HCA=%q", env["NCCL_IB_HCA"])
+	}
+	if env["NCCL_IB_DISABLE"] != "0" {
+		t.Errorf("NCCL_IB_DISABLE=%q with a fabric present", env["NCCL_IB_DISABLE"])
+	}
+	// Bootstrap must stay on the management network; NCCL otherwise probes
+	// every interface, including IB-over-IP that cannot route between fabrics.
+	if env["NCCL_SOCKET_IFNAME"] != "eth0" || env["GLOO_SOCKET_IFNAME"] != "eth0" {
+		t.Errorf("socket interfaces not pinned: %q / %q",
+			env["NCCL_SOCKET_IFNAME"], env["GLOO_SOCKET_IFNAME"])
+	}
+}
+
+func TestProfileEnvOverridesGenerated(t *testing.T) {
+	s := multiNodeSpec(0)
+	s.Profile.Env = map[string]string{
+		"NCCL_IB_HCA":        "mlx5_9",
+		"NCCL_NET_GDR_LEVEL": "SYS",
+	}
+	cmd := buildEngine(t, &VLLM{}, s)
+	env := envMap(cmd.Env)
+
+	if env["NCCL_IB_HCA"] != "mlx5_9" {
+		t.Error("a profile override did not win; it is the escape hatch for per-cluster tuning")
+	}
+	if env["NCCL_NET_GDR_LEVEL"] != "SYS" {
+		t.Error("an extra profile variable was dropped")
+	}
+}
+
+func TestProcessEnvironmentIsInherited(t *testing.T) {
+	// Engines need PATH, HF_HOME, LD_LIBRARY_PATH and whatever else the unit
+	// file set up.
+	t.Setenv("LLMGW_TEST_MARKER", "present")
+	cmd := buildOne(t, &VLLM{}, singleNodeSpec())
+	if envMap(cmd.Env)["LLMGW_TEST_MARKER"] != "present" {
+		t.Error("the process environment was not inherited")
+	}
+}
+
+func TestEnvIsSorted(t *testing.T) {
+	cmd := buildOne(t, &VLLM{}, singleNodeSpec())
+	if !slices.IsSorted(cmd.Env) {
+		t.Error("environment is not sorted; launches would not be reproducible in logs")
+	}
+}
+
+// --- vLLM ---
+
+func TestVLLMSingleNodeArgs(t *testing.T) {
+	cmd := buildOne(t, &VLLM{cfg: VLLMConfig{Bin: "/opt/vllm/bin/vllm"}}, singleNodeSpec())
+	if cmd.Path != "/opt/vllm/bin/vllm" {
+		t.Errorf("Path=%q", cmd.Path)
+	}
+	if cmd.Args[0] != "serve" || cmd.Args[1] != "/models/llama" {
+		t.Errorf("args start wrong: %v", cmd.Args[:2])
+	}
+	if argValue(cmd.Args, "--tensor-parallel-size") != "4" {
+		t.Errorf("TP not passed: %v", cmd.Args)
+	}
+	if argValue(cmd.Args, "--port") != "20000" {
+		t.Errorf("port not passed: %v", cmd.Args)
+	}
+	// Single node means no distributed backend flag at all.
+	if slices.Contains(cmd.Args, "--distributed-executor-backend") {
+		t.Error("a distributed backend was selected for a single-node profile")
+	}
+}
+
+func TestVLLMRayLeaderRunsTwoProcesses(t *testing.T) {
+	// The leader starts a Ray head and then the server against it. A single
+	// command would mean running vLLM against a cluster that does not exist.
+	v := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
+	cmds, err := v.Commands(multiNodeSpec(0))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(cmds) != 2 {
+		t.Fatalf("expected 2 commands for the Ray leader, got %d", len(cmds))
+	}
+	if cmds[0].Name != "ray" || !slices.Contains(cmds[0].Args, "--head") {
+		t.Errorf("first command is not the Ray head: %+v", cmds[0])
+	}
+	if !cmds[0].WaitForLeaderPort {
+		t.Error("the server would start before the head is listening")
+	}
+	if cmds[1].Name != "vllm" || argValue(cmds[1].Args, "--distributed-executor-backend") != "ray" {
+		t.Errorf("second command is not the vLLM server: %+v", cmds[1])
+	}
+	if got := envMap(cmds[1].Env)["RAY_ADDRESS"]; got != "10.0.0.10:20001" {
+		t.Errorf("RAY_ADDRESS=%q", got)
+	}
+}
+
+func TestVLLMRayWorkerRunsOnlyRay(t *testing.T) {
+	// A Ray worker runs no vLLM process: the cluster hands its GPUs to the
+	// leader. Starting a second server there would bind a port nobody uses and
+	// compete for the same cards.
+	v := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
+	cmds, err := v.Commands(multiNodeSpec(1))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(cmds) != 1 {
+		t.Fatalf("expected 1 command for a Ray worker, got %d: %+v", len(cmds), cmds)
+	}
+	if cmds[0].Name != "ray" || slices.Contains(cmds[0].Args, "--head") {
+		t.Errorf("worker command wrong: %+v", cmds[0])
+	}
+	if argValue(cmds[0].Args, "--address") != "10.0.0.10:20001" {
+		t.Errorf("worker is not joining the leader: %v", cmds[0].Args)
+	}
+}
+
+func TestVLLMMPMode(t *testing.T) {
+	v := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}
+	cmd := buildOne(t, v, multiNodeSpec(0))
+	if argValue(cmd.Args, "--distributed-executor-backend") != "mp" {
+		t.Errorf("backend not set: %v", cmd.Args)
+	}
+	env := envMap(cmd.Env)
+	if env["VLLM_HOST_IP"] != "10.0.0.10" || env["VLLM_PORT"] != "20001" {
+		t.Errorf("dist init not configured: %q %q", env["VLLM_HOST_IP"], env["VLLM_PORT"])
+	}
+}
+
+func TestVLLMUnknownLaunchMode(t *testing.T) {
+	v := &VLLM{cfg: VLLMConfig{LaunchMode: "sorcery"}}
+	if _, err := v.Commands(multiNodeSpec(0)); err == nil {
+		t.Error("an unknown launch mode was accepted")
+	}
+}
+
+func TestOnlyLeaderServesHTTP(t *testing.T) {
+	// The fix for the bug that stopped multi-node replicas coming up at all:
+	// health-checking a worker's port times out and fails a healthy replica.
+	for _, e := range []Engine{&VLLM{}, &SGLang{}} {
+		if !e.ServesHTTP(multiNodeSpec(0)) {
+			t.Errorf("%s: the leader must serve HTTP", e.Name())
+		}
+		if e.ServesHTTP(multiNodeSpec(1)) {
+			t.Errorf("%s: a worker must not be expected to serve HTTP", e.Name())
+		}
+		if !e.ServesHTTP(singleNodeSpec()) {
+			t.Errorf("%s: a single-node shard must serve HTTP", e.Name())
+		}
+	}
+}
+
+func TestNoRayForSingleNodeOrMPMode(t *testing.T) {
+	ray := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
+	if cmds := buildOne(t, ray, singleNodeSpec()); cmds.Name != "vllm" {
+		t.Errorf("a single-node profile was given a Ray process: %+v", cmds)
+	}
+	mp := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}
+	if cmds := buildOne(t, mp, multiNodeSpec(0)); cmds.Name != "vllm" {
+		t.Errorf("mp mode was given a Ray process: %+v", cmds)
+	}
+}
+
+// --- SGLang ---
+
+func TestSGLangMultiNodeArgs(t *testing.T) {
+	cmd := buildOne(t, &SGLang{cfg: SGLangConfig{Python: "/opt/sglang/bin/python"}}, multiNodeSpec(1))
+	if cmd.Path != "/opt/sglang/bin/python" {
+		t.Errorf("Path=%q", cmd.Path)
+	}
+	if cmd.Args[0] != "-m" || cmd.Args[1] != "sglang.launch_server" {
+		t.Errorf("not launched as a module: %v", cmd.Args[:2])
+	}
+	if argValue(cmd.Args, "--nnodes") != "2" || argValue(cmd.Args, "--node-rank") != "1" {
+		t.Errorf("multi-node flags wrong: %v", cmd.Args)
+	}
+	if argValue(cmd.Args, "--dist-init-addr") != "10.0.0.10:20001" {
+		t.Errorf("dist-init-addr wrong: %v", cmd.Args)
+	}
+}
+
+func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) {
+	cmd := buildOne(t, &SGLang{}, singleNodeSpec())
+	for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {
+		if slices.Contains(cmd.Args, flag) {
+			t.Errorf("%s was passed for a single-node profile", flag)
+		}
+	}
+}
+
+func TestProfileArgsComeLast(t *testing.T) {
+	// Operators must be able to override anything the adapter generated.
+	s := singleNodeSpec()
+	s.Profile.Args = []string{"--max-model-len", "32768"}
+	cmd := buildEngine(t, &VLLM{}, s)
+
+	i := slices.Index(cmd.Args, "--max-model-len")
+	j := slices.Index(cmd.Args, "--tensor-parallel-size")
+	if i < j {
+		t.Errorf("profile args came before generated ones: %v", cmd.Args)
+	}
+}
+
+// --- validation ---
+
+func h100(n int) []GPUInfo {
+	out := make([]GPUInfo, n)
+	for i := range out {
+		out[i] = GPUInfo{Index: i, Name: "NVIDIA H100 80GB HBM3", ComputeCapability: "9.0", MemoryTotalMB: 81559}
+	}
+	return out
+}
+
+func TestValidateAcceptsMatchingProfile(t *testing.T) {
+	p := Profile{Name: "ok", TP: 4, ModelPath: "/m"}
+	if err := (&VLLM{}).Validate(p, h100(4)); err != nil {
+		t.Errorf("a valid profile was rejected: %v", err)
+	}
+}
+
+func TestValidateRejectsMixedGPUTypes(t *testing.T) {
+	// The invariant from plan §1.1: a TP group never mixes GPU types.
+	gpus := h100(2)
+	gpus[1].Name = "NVIDIA A100-SXM4-40GB"
+
+	err := (&VLLM{}).Validate(Profile{Name: "mixed", TP: 2, ModelPath: "/m"}, gpus)
+	if err == nil || !strings.Contains(err.Error(), "may not mix GPU types") {
+		t.Errorf("mixed GPU types accepted: %v", err)
+	}
+}
+
+func TestValidateRejectsFP8OnOldHardware(t *testing.T) {
+	gpus := h100(4)
+	for i := range gpus {
+		gpus[i].Name = "NVIDIA A100-SXM4-80GB"
+		gpus[i].ComputeCapability = "8.0"
+	}
+	p := Profile{Name: "fp8", TP: 4, ModelPath: "/m", Args: []string{"--quantization", "fp8"}}
+
+	err := (&VLLM{}).Validate(p, gpus)
+	if err == nil || !strings.Contains(err.Error(), "compute capability") {
+		t.Errorf("fp8 accepted on compute capability 8.0: %v", err)
+	}
+
+	// The same profile is fine on Hopper.
+	if err := (&VLLM{}).Validate(p, h100(4)); err != nil {
+		t.Errorf("fp8 rejected on H100: %v", err)
+	}
+}
+
+func TestValidateRejectsInconsistentParallelism(t *testing.T) {
+	// 2 nodes x 8 GPUs cannot serve TP 4.
+	p := Profile{Name: "bad", TP: 4, Nodes: 2, GPUsPerNode: 8, ModelPath: "/m"}
+	if err := (&VLLM{}).Validate(p, h100(8)); err == nil {
+		t.Error("inconsistent node/GPU/TP arithmetic accepted")
+	}
+}
+
+func TestValidateRejectsTooFewGPUs(t *testing.T) {
+	if err := (&VLLM{}).Validate(Profile{Name: "big", TP: 8, ModelPath: "/m"}, h100(4)); err == nil {
+		t.Error("a TP 8 profile was accepted on a 4-GPU node")
+	}
+}
+
+func TestValidateRequiresModelPath(t *testing.T) {
+	if err := (&VLLM{}).Validate(Profile{Name: "nopath", TP: 1}, h100(1)); err == nil {
+		t.Error("a profile without a model path was accepted")
+	}
+}
+
+func TestRegistry(t *testing.T) {
+	r := NewRegistry(Config{})
+	if got := r.Names(); !slices.Equal(got, []string{"llamacpp", "sglang", "sim", "vllm"}) {
+		t.Errorf("registry contents: %v", got)
+	}
+	if _, err := r.Get("vllm"); err != nil {
+		t.Errorf("vllm missing: %v", err)
+	}
+	err := func() error { _, err := r.Get("nope"); return err }()
+	if err == nil || !strings.Contains(err.Error(), "llamacpp, sglang, sim, vllm") {
+		t.Errorf("an unknown engine should list what is available: %v", err)
+	}
+}
+
+func TestQuantizationOf(t *testing.T) {
+	for _, args := range [][]string{
+		{"--quantization", "fp8"},
+		{"--quantization=fp8"},
+		{"--max-model-len", "100", "--quantization", "fp8"},
+	} {
+		if got := quantizationOf(args); got != "fp8" {
+			t.Errorf("quantizationOf(%v)=%q", args, got)
+		}
+	}
+	if got := quantizationOf([]string{"--quantization"}); got != "" {
+		t.Errorf("a dangling flag should yield nothing, got %q", got)
+	}
+}

internal/agent/engines/llamacpp.go

@@ -0,0 +1,197 @@
+package engines
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"sync/atomic"
+)
+
+// LlamaCppConfig locates the llama.cpp installation.
+type LlamaCppConfig struct {
+	// Bin is the llama-server binary. It is the server, not the CLI: the
+	// gateway talks OpenAI over HTTP and needs the one that serves it.
+	// Empty means the agent installs llama.cpp itself, at the version the
+	// control plane asks for. Setting it opts out of that.
+	Bin string `koanf:"bin"`
+	// Parallel is how many requests a server handles at once, when a profile
+	// does not say. llama.cpp defaults to one, which would make every second
+	// request wait behind the first no matter what the router decided.
+	Parallel int `koanf:"parallel"`
+}
+
+// LlamaCpp builds llama.cpp launch commands.
+//
+// It is the engine that makes a cluster without GPUs useful: llama.cpp runs a
+// small model on CPU at a speed that is fine for testing the whole path, and it
+// serves the same OpenAI API as the others, so nothing downstream can tell the
+// difference.
+type LlamaCpp struct {
+	cfg       LlamaCppConfig
+	installed atomic.Pointer[string]
+}
+
+// Name identifies the engine in the catalog.
+func (*LlamaCpp) Name() string { return "llamacpp" }
+
+// HealthPath is llama.cpp's readiness endpoint. It returns 503 while the model
+// is still loading, which is exactly what the supervisor needs.
+func (*LlamaCpp) HealthPath() string { return "/health" }
+
+// MetricsPath is llama.cpp's Prometheus endpoint, which only exists when the
+// server was started with --metrics. Commands always passes it.
+func (*LlamaCpp) MetricsPath() string { return "/metrics" }
+
+// ServesHTTP reports whether this rank answers the API. Every llama.cpp shard
+// does, because a llama.cpp replica is always exactly one shard.
+func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true }
+
+// Validate checks the profile against the GPUs it would run on.
+//
+// The two refusals here are about not pretending: llama.cpp has no equivalent
+// of a tensor-parallel group spanning machines, and its multi-GPU support is a
+// different mechanism (--split-mode, --tensor-split) that this adapter does not
+// drive. Accepting either would produce a replica that starts and then serves
+// from one machine while the catalog believes it has several.
+func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error {
+	if err := validateCommon(p, gpus); err != nil {
+		return err
+	}
+	if p.Nodes > 1 {
+		return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica",
+			p.Name, p.Nodes)
+	}
+	if p.TP > 1 {
+		return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+
+			"it splits a model across local GPUs with --tensor-split, which this adapter does not drive",
+			p.Name)
+	}
+	return nil
+}
+
+// Commands returns the processes for one shard, which is always one process.
+func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
+	bin := l.bin()
+	p := s.Profile
+
+	model, err := ggufIn(p.ModelPath)
+	if err != nil {
+		return nil, err
+	}
+	args := []string{
+		"--model", model,
+		"--host", "0.0.0.0",
+		"--port", strconv.Itoa(s.HTTPPort),
+		// Without this there is no /metrics, and an endpoint the router cannot
+		// measure is one it has to route to blind.
+		"--metrics",
+	}
+	// The model's own chat template rather than a generic one, which is what
+	// renders tool definitions in the form the model was trained to read.
+	//
+	// Pinned rather than relied on: build 10991 has it on by default, older
+	// builds do not, and an engine upgrade should not be able to change how
+	// prompts are built underneath a running fleet. Passing it explicitly was
+	// a no-op on the build here — it was added while chasing tool calls that
+	// came back as text, and did not fix that, which turned out to be the
+	// model rather than the template.
+	if !hasFlag(p.Args, "--jinja", "--no-jinja") {
+		args = append(args, "--jinja")
+	}
+	if p.ServedModelName != "" {
+		args = append(args, "--alias", p.ServedModelName)
+	}
+	if !hasFlag(p.Args, "--parallel", "-np") {
+		n := l.cfg.Parallel
+		if n <= 0 {
+			n = 4
+		}
+		args = append(args, "--parallel", strconv.Itoa(n))
+	}
+	// With GPUs assigned, offload everything; llama.cpp keeps the model on the
+	// CPU otherwise, which is not what a GPU placement asked for. Without them
+	// the flag is omitted entirely rather than set to zero, so a build with no
+	// GPU support does not see an argument it rejects.
+	if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") {
+		args = append(args, "--n-gpu-layers", "999")
+	}
+	args = append(args, p.Args...)
+
+	return []Command{{
+		Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
+	}}, nil
+}
+
+// hasFlag reports whether the profile already sets one of these flags, in
+// either "--flag value" or "--flag=value" form.
+func hasFlag(args []string, names ...string) bool {
+	for _, a := range args {
+		for _, n := range names {
+			if a == n || strings.HasPrefix(a, n+"=") {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// ggufIn resolves a model path that may be a directory.
+//
+// A multi-file version arrives as the directory it was downloaded into, because
+// that is the only shape a set can take. llama.cpp wants a file: given the
+// first part of a split GGUF it finds the rest beside it, so the first part is
+// what it gets.
+//
+// A path that is already a file is returned untouched, which is every
+// single-file model and every hand-staged one.
+func ggufIn(path string) (string, error) {
+	st, err := os.Stat(path)
+	if err != nil || !st.IsDir() {
+		// Not a directory, or not there at all — the second is the engine's to
+		// report, with the message it would have given anyway.
+		return path, nil //nolint:nilerr // a missing path is the engine's error to give
+	}
+	entries, err := os.ReadDir(path)
+	if err != nil {
+		return "", err
+	}
+	var names []string
+	for _, e := range entries {
+		if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), ".gguf") {
+			names = append(names, e.Name())
+		}
+	}
+	if len(names) == 0 {
+		return "", fmt.Errorf("%s holds no .gguf file", path)
+	}
+	// Sorted, so a split model's first part comes first: the parts are named
+	// "-00001-of-00004" and lexical order is shard order.
+	sort.Strings(names)
+	return filepath.Join(path, names[0]), nil
+}
+
+// UseInstalled points the engine at a build the agent installed and checked.
+// It takes precedence over everything else, because it is the only one that
+// was seen to run.
+func (l *LlamaCpp) UseInstalled(path string) { l.installed.Store(&path) }
+
+// Configured reports whether the agent's configuration names a binary. A
+// named binary is the operator's deliberate choice; one merely found on PATH
+// is not, and is not reported as installed.
+func (l *LlamaCpp) Configured() bool { return l.cfg.Bin != "" }
+
+// bin is the llama-server to run: the installed build, else the configured
+// one, else whatever PATH has — which is only ever reached by a node not
+// provisioned by the agent and not told where its engine is.
+func (l *LlamaCpp) bin() string {
+	if p := l.installed.Load(); p != nil {
+		return *p
+	}
+	if l.cfg.Bin != "" {
+		return l.cfg.Bin
+	}
+	return "llama-server"
+}

internal/agent/engines/llamacpp_test.go

@@ -0,0 +1,247 @@
+package engines
+
+import (
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+)
+
+func llamaSpec(gpus []int) ShardSpec {
+	return ShardSpec{
+		ReplicaID: "rep-1", ShardID: "sh-1", Rank: 0, NNodes: 1,
+		GPUIndices: gpus, HTTPPort: 8000, MgmtInterface: "eth0",
+		Profile: Profile{
+			Name: "cpu", Engine: "llamacpp", TP: 1,
+			ModelPath: "/models/qwen.gguf", ServedModelName: "qwen-0.5b",
+		},
+	}
+}
+
+func TestLlamaCppCommand(t *testing.T) {
+	cmds, err := (&LlamaCpp{}).Commands(llamaSpec(nil))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(cmds) != 1 {
+		t.Fatalf("a llama.cpp shard is one process, got %d", len(cmds))
+	}
+	c := cmds[0]
+	if c.Path != "llama-server" {
+		t.Errorf("path %q", c.Path)
+	}
+
+	// --metrics is not optional: llama-server serves no /metrics without it, and
+	// an endpoint the router cannot measure is one it routes to blind.
+	if !slices.Contains(c.Args, "--metrics") {
+		t.Error("the server was started without --metrics")
+	}
+	if v, _ := flagValue(c.Args, "--model"); v != "/models/qwen.gguf" {
+		t.Errorf("--model %q", v)
+	}
+	if v, _ := flagValue(c.Args, "--port"); v != "8000" {
+		t.Errorf("--port %q", v)
+	}
+	if v, _ := flagValue(c.Args, "--alias"); v != "qwen-0.5b" {
+		t.Errorf("--alias %q", v)
+	}
+	// llama.cpp serves one request at a time unless told otherwise, which would
+	// make every second request wait no matter what the router decided.
+	if v, ok := flagValue(c.Args, "--parallel"); !ok || v == "1" {
+		t.Errorf("--parallel %q", v)
+	}
+	if slices.Contains(c.Args, "--n-gpu-layers") {
+		t.Error("a CPU placement was given GPU layers")
+	}
+}
+
+func TestLlamaCppOffloadsWhenGivenGPUs(t *testing.T) {
+	cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
+	if v, ok := flagValue(cmds[0].Args, "--n-gpu-layers"); !ok || v != "999" {
+		t.Errorf("a GPU placement should offload the model, got %q", v)
+	}
+	// The shard must still be pinned to its own card.
+	if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=2") {
+		t.Error("the shard was not pinned to its GPU")
+	}
+}
+
+func TestProfileArgumentsWinOverDefaults(t *testing.T) {
+	// The escape hatch has to actually work, or an operator tuning slot count
+	// ends up with two --parallel flags and whichever llama.cpp happens to
+	// prefer.
+	s := llamaSpec(nil)
+	s.Profile.Args = []string{"--parallel", "16", "--ctx-size", "4096"}
+
+	cmds, _ := (&LlamaCpp{}).Commands(s)
+	n := 0
+	for _, a := range cmds[0].Args {
+		if a == "--parallel" {
+			n++
+		}
+	}
+	if n != 1 {
+		t.Errorf("--parallel appears %d times", n)
+	}
+	if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "16" {
+		t.Errorf("the profile's value lost: %q", v)
+	}
+}
+
+func TestShortFormArgumentsAreRecognized(t *testing.T) {
+	s := llamaSpec([]int{0})
+	s.Profile.Args = []string{"-np", "8", "-ngl", "20"}
+
+	cmds, _ := (&LlamaCpp{}).Commands(s)
+	for _, flag := range []string{"--parallel", "--n-gpu-layers"} {
+		if slices.Contains(cmds[0].Args, flag) {
+			t.Errorf("%s was added although the profile set its short form", flag)
+		}
+	}
+}
+
+func TestLlamaCppRefusesWhatItCannotDo(t *testing.T) {
+	// Accepting either of these would produce a replica that starts and then
+	// serves from one machine while the catalog believes it has several.
+	multi := Profile{Name: "multi", Engine: "llamacpp", TP: 1, Nodes: 2, GPUsPerNode: 1, ModelPath: "/m"}
+	if err := (&LlamaCpp{}).Validate(multi, nil); err == nil {
+		t.Error("a multi-node llama.cpp profile was accepted")
+	}
+
+	tp := Profile{Name: "tp", Engine: "llamacpp", TP: 2, GPUsPerNode: 2, ModelPath: "/m"}
+	if err := (&LlamaCpp{}).Validate(tp, h100(2)); err == nil {
+		t.Error("a tensor-parallel llama.cpp profile was accepted")
+	}
+}
+
+func TestCPUProfileValidates(t *testing.T) {
+	p := Profile{Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf"}
+	if err := (&LlamaCpp{}).Validate(p, nil); err != nil {
+		t.Errorf("a CPU profile on a node with no GPUs was rejected: %v", err)
+	}
+}
+
+func TestLlamaCppAlwaysServesHTTP(t *testing.T) {
+	// Unlike the others there is no worker rank to get this wrong for.
+	if !(&LlamaCpp{}).ServesHTTP(llamaSpec(nil)) {
+		t.Error("a llama.cpp shard must answer the API")
+	}
+}
+
+func TestBinaryIsConfigurable(t *testing.T) {
+	l := &LlamaCpp{cfg: LlamaCppConfig{Bin: "/opt/llama.cpp/bin/llama-server", Parallel: 2}}
+	cmds, _ := l.Commands(llamaSpec(nil))
+	if cmds[0].Path != "/opt/llama.cpp/bin/llama-server" {
+		t.Errorf("path %q", cmds[0].Path)
+	}
+	if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "2" {
+		t.Errorf("--parallel %q", v)
+	}
+}
+
+func TestUnknownEngineNamesLlamaCpp(t *testing.T) {
+	_, err := NewRegistry(Config{}).Get("llama.cpp")
+	if err == nil || !strings.Contains(err.Error(), "llamacpp") {
+		t.Errorf("the suggestion should name the real engine: %v", err)
+	}
+}
+
+// flagValue reads a "--flag value" pair out of a command line.
+func flagValue(args []string, flag string) (string, bool) {
+	for i, a := range args {
+		if a == flag && i+1 < len(args) {
+			return args[i+1], true
+		}
+	}
+	return "", false
+}
+
+func TestLlamaCppUsesTheModelsChatTemplate(t *testing.T) {
+	// Pinned so an engine upgrade cannot change how prompts are built under a
+	// running fleet: build 10991 defaults to jinja, older builds do not.
+	//
+	// Not a fix for tool calling, though it was added while chasing that. On
+	// this build the flag is a no-op and the behaviour is identical with and
+	// without it.
+	l := &LlamaCpp{}
+	cmds, err := l.Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: "/m.gguf"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Contains(cmds[0].Args, "--jinja") {
+		t.Errorf("--jinja is missing; tool calls will come back as text: %v", cmds[0].Args)
+	}
+}
+
+func TestLlamaCppLetsAProfileTurnTheTemplateOff(t *testing.T) {
+	// A model whose bundled template is broken has to be serviceable without
+	// editing the adapter.
+	l := &LlamaCpp{}
+	cmds, err := l.Commands(ShardSpec{
+		HTTPPort: 8080,
+		Profile:  Profile{ModelPath: "/m.gguf", Args: []string{"--no-jinja"}},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if slices.Contains(cmds[0].Args, "--jinja") {
+		t.Errorf("the profile said --no-jinja and got both: %v", cmds[0].Args)
+	}
+}
+
+func TestASetDirectoryResolvesToItsFirstPart(t *testing.T) {
+	dir := t.TempDir()
+	// Deliberately out of order on disk, because readdir order is not shard
+	// order and the first part is the one llama.cpp must be given.
+	for _, n := range []string{
+		"qwen-q4_k_m-00003-of-00003.gguf",
+		"qwen-q4_k_m-00001-of-00003.gguf",
+		"qwen-q4_k_m-00002-of-00003.gguf",
+	} {
+		if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	got, err := ggufIn(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Given the first part llama.cpp finds the rest beside it; given the third
+	// it loads a third of a model.
+	if filepath.Base(got) != "qwen-q4_k_m-00001-of-00003.gguf" {
+		t.Errorf("expected the first shard, got %s", filepath.Base(got))
+	}
+}
+
+func TestAPlainFilePathIsUntouched(t *testing.T) {
+	// Every single-file model and every hand-staged one takes this path.
+	p := filepath.Join(t.TempDir(), "model.gguf")
+	if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if got, _ := ggufIn(p); got != p {
+		t.Errorf("expected the path unchanged, got %s", got)
+	}
+}
+
+func TestADirectoryWithNoWeightsSaysSo(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ggufIn(dir); err == nil {
+		t.Error("a directory with no .gguf in it is not a model llama.cpp can load")
+	}
+}
+
+func TestAMissingPathIsLeftToTheEngine(t *testing.T) {
+	// The engine's own "no such file" names the path and the flag it came
+	// from, which is more use than anything this could say.
+	p := "/nonexistent/model.gguf"
+	got, err := ggufIn(p)
+	if err != nil || got != p {
+		t.Errorf("expected it passed through, got %q %v", got, err)
+	}
+}

internal/agent/engines/probe.go

@@ -0,0 +1,224 @@
+package engines
+
+import (
+	"context"
+	"errors"
+	"os/exec"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+)
+
+// Unknown is the version reported for an engine that is installed and would
+// not say which version it is.
+//
+// A value, not an absent key, because the two mean different things and
+// placement reads the keys: absent is "this node cannot run it", Unknown is
+// "it is here and the probe could not get a number out of it". Collapsing them
+// would make a slow engine look like a missing one and take a node out of
+// scheduling for being slow to answer --version.
+const Unknown = "unknown"
+
+// Prober is an engine that can say whether it is installed on this machine.
+//
+// Optional, and asserted for rather than part of Engine: an adapter that has no
+// way to check should report nothing rather than be forced to invent an answer.
+type Prober interface {
+	// Probe returns the engine's version, or an error when it is not installed.
+	Probe(ctx context.Context) (string, error)
+}
+
+// Probe asks every registered engine whether it is here.
+//
+// The result is what the node reports at registration, and its key set is what
+// placement filters on. Until this existed nobody knew: the registry registers
+// every adapter whether or not its binary exists, so a node with no vLLM
+// accepted a vLLM shard, registered healthy, and failed the shard minutes later
+// with "executable file not found" — in the engine's own words, on a machine
+// nobody was watching.
+//
+// Bounded, because it runs on the path to registering and an agent that cannot
+// reach the control plane is worse than one that reports an incomplete list.
+func (r *Registry) Probe(ctx context.Context) map[string]string {
+	out := map[string]string{}
+	for name, e := range r.engines {
+		p, ok := e.(Prober)
+		if !ok {
+			continue
+		}
+		v, err := p.Probe(ctx)
+		if err != nil {
+			continue
+		}
+		if v == "" {
+			v = Unknown
+		}
+		out[name] = v
+	}
+	return out
+}
+
+// Names of the engines in a probe result, sorted.
+func ProbedNames(m map[string]string) []string {
+	out := make([]string, 0, len(m))
+	for n := range m {
+		out = append(out, n)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// probeTimeout bounds one engine's version check.
+//
+// Generous, because vLLM's --version imports torch and that is seconds on a
+// cold page cache, and mean, because this is startup: an engine that has not
+// answered in this long is reported as present-with-unknown-version rather
+// than waited for.
+const probeTimeout = 8 * time.Second
+
+// run executes a version command, saying whether the program was even there.
+//
+// The two failures are different answers and the caller has to tell them
+// apart. A binary that is absent is an engine this node cannot run. A binary
+// that is present and exits non-zero is an engine it probably can — unless the
+// command was a test of whether a module imports, where a non-zero exit is the
+// whole answer.
+func run(ctx context.Context, bin string, args ...string) (out string, found bool, ok bool) {
+	path, err := exec.LookPath(bin)
+	if err != nil {
+		return "", false, false
+	}
+	ctx, cancel := context.WithTimeout(ctx, probeTimeout)
+	defer cancel()
+
+	// Combined, not just stdout: engines print their version to either, and one
+	// that chose stderr would otherwise read as having no version at all.
+	b, err := exec.CommandContext(ctx, path, args...).CombinedOutput()
+	return string(b), true, err == nil
+}
+
+// installedBinary reports an engine that is its own executable.
+//
+// It resolves the path and runs nothing. The first version of this ran the
+// engine with --version, which is fine for an engine and not fine for the
+// wrapper scripts people put in front of one: a script that shifts past flags
+// it does not recognise and then execs the server will start an inference
+// server, at agent startup, on every node. The agent's own test suite installs
+// exactly such a script, and caught it.
+//
+// So the version is not read here. Presence is what placement needs, and what
+// the version is for — telling whether an engine still accepts the flags llmgw
+// builds — is `llmgw ctl engine-check`, which is run deliberately by somebody
+// who meant to run the engine.
+func installedBinary(bin string) (string, error) {
+	if _, err := exec.LookPath(bin); err != nil {
+		return "", errNotInstalled
+	}
+	return Unknown, nil
+}
+
+// versionOfModule probes an engine that is a module inside an interpreter.
+//
+// This one does execute, and safely: the command is llmgw's own one-line
+// import, not the engine's launcher, so there is no wrapper to shift past
+// flags and start a server. There is also no alternative — a module has no
+// path to look up.
+//
+// A non-zero exit is the answer: python3 exists on almost every machine, so
+// presence of the interpreter says nothing. Treating a failed import as
+// "installed" reported SGLang on every node in a fleet that had none, which
+// would take nodes into scheduling for an engine they cannot run — the exact
+// failure this whole change exists to stop.
+func versionOfModule(ctx context.Context, python string, args ...string) (string, error) {
+	out, found, ok := run(ctx, python, args...)
+	if !found || !ok {
+		return "", errNotInstalled
+	}
+	return firstVersion(out), nil
+}
+
+var errNotInstalled = errors.New("not installed on this node")
+
+var versionPattern = regexp.MustCompile(`\b\d+\.\d+(\.\d+)?\S*`)
+
+// firstVersion is the first thing in the output that looks like a version.
+//
+// Pattern rather than parsing each engine's format: they print "vllm 0.6.3",
+// "version: b4123 (…)", and a paragraph with the number in the middle, and all
+// of those change between releases. What this is for is telling one build from
+// another, which the first number does.
+func firstVersion(s string) string {
+	for _, line := range strings.Split(s, "\n") {
+		if m := versionPattern.FindString(line); m != "" {
+			return strings.TrimRight(m, ",;)")
+		}
+	}
+	return Unknown
+}
+
+// Configured is an engine whose location the agent's configuration names.
+type Configured interface {
+	Configured() bool
+}
+
+// ProbeConfigured probes only the engines the configuration names.
+//
+// Probe looks everywhere — PATH, the default interpreter — and reports what
+// it finds, which is exactly the trust in whatever happens to be on a machine
+// that installing engines replaces. A node the agent provisions reports what
+// it installed, plus what the operator named on purpose; nothing else.
+func (r *Registry) ProbeConfigured(ctx context.Context) map[string]string {
+	out := map[string]string{}
+	for name, e := range r.engines {
+		c, ok := e.(Configured)
+		if !ok || !c.Configured() {
+			continue
+		}
+		p, ok := e.(Prober)
+		if !ok {
+			continue
+		}
+		if v, err := p.Probe(ctx); err == nil {
+			if v == "" {
+				v = Unknown
+			}
+			out[name] = v
+		}
+	}
+	return out
+}
+
+// Probe reports whether the llama.cpp server is installed.
+func (l *LlamaCpp) Probe(context.Context) (string, error) {
+	return installedBinary(l.bin())
+}
+
+// Configured reports whether the configuration names vLLM.
+func (v *VLLM) Configured() bool { return v.cfg.Bin != "" }
+
+// Configured reports whether the configuration names SGLang's interpreter.
+func (g *SGLang) Configured() bool { return g.cfg.Python != "" }
+
+// Probe reports whether vLLM is installed.
+func (v *VLLM) Probe(context.Context) (string, error) {
+	bin := v.cfg.Bin
+	if bin == "" {
+		bin = "vllm"
+	}
+	return installedBinary(bin)
+}
+
+// Probe reports the SGLang version.
+//
+// Through the interpreter, because SGLang is a module rather than a binary:
+// asking the shell for "sglang" finds nothing on a machine that can run it
+// perfectly well.
+func (g *SGLang) Probe(ctx context.Context) (string, error) {
+	python := g.cfg.Python
+	if python == "" {
+		python = "python3"
+	}
+	return versionOfModule(ctx, python, "-c",
+		"import sglang, sys; sys.stdout.write(getattr(sglang, '__version__', ''))")
+}

internal/agent/engines/probe_test.go

@@ -0,0 +1,107 @@
+package engines
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// script writes an executable that stands in for an engine.
+func script(t *testing.T, name, body string) string {
+	t.Helper()
+	dir := t.TempDir()
+	p := filepath.Join(dir, name)
+	if err := os.WriteFile(p, []byte("#!/bin/sh\n"+body+"\n"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	return p
+}
+
+func TestAnEngineThatIsNotInstalledIsAbsentFromTheReport(t *testing.T) {
+	r := NewRegistry(Config{})
+	r.engines = map[string]Engine{
+		"vllm": &VLLM{cfg: VLLMConfig{Bin: "/nonexistent/vllm"}},
+	}
+	// Absent, not present-with-unknown-version. Placement reads the keys, and
+	// a node listed for an engine it cannot run is the failure being fixed.
+	if got := r.Probe(t.Context()); len(got) != 0 {
+		t.Errorf("expected nothing, got %v", got)
+	}
+}
+
+func TestProbingNeverRunsTheEngine(t *testing.T) {
+	// A wrapper that shifts past flags it does not recognise and then starts
+	// the server. People put these in front of engines, and the agent's own
+	// tests install one. Probing with --version would launch an inference
+	// server at agent startup, on every node.
+	marker := filepath.Join(t.TempDir(), "ran")
+	bin := script(t, "fake-vllm", "touch "+marker+"; sleep 60")
+
+	r := NewRegistry(Config{})
+	r.engines = map[string]Engine{"vllm": &VLLM{cfg: VLLMConfig{Bin: bin}}}
+
+	got := r.Probe(t.Context())
+	if _, ok := got["vllm"]; !ok {
+		t.Error("a binary that is there should be reported present")
+	}
+	if _, err := os.Stat(marker); err == nil {
+		t.Fatal("the engine was executed by a probe")
+	}
+}
+
+func TestAModuleThatWillNotImportIsNotInstalled(t *testing.T) {
+	// python3 is on almost every machine, so the interpreter being there says
+	// nothing about SGLang. Reporting it present put every node in a fleet
+	// forward for an engine none of them had.
+	bin := script(t, "python3", "exit 1")
+	r := NewRegistry(Config{})
+	r.engines = map[string]Engine{"sglang": &SGLang{cfg: SGLangConfig{Python: bin}}}
+
+	if got := r.Probe(t.Context()); len(got) != 0 {
+		t.Errorf("a failed import means not installed, got %v", got)
+	}
+}
+
+func TestAModuleThatImportsIsReportedWithItsVersion(t *testing.T) {
+	bin := script(t, "python3", `echo "0.4.1"`)
+	r := NewRegistry(Config{})
+	r.engines = map[string]Engine{"sglang": &SGLang{cfg: SGLangConfig{Python: bin}}}
+
+	got := r.Probe(t.Context())
+	if got["sglang"] != "0.4.1" {
+		t.Errorf("expected the version, got %v", got)
+	}
+}
+
+func TestAVersionIsPickedOutOfWhateverIsPrinted(t *testing.T) {
+	// Only the module probe reaches this now, and a Python package version is
+	// dotted. A bare build number is deliberately not matched: it would turn a
+	// stray number in a line of prose into a version.
+	for in, want := range map[string]string{
+		"0.4.1":                 "0.4.1",
+		"sglang 0.4.post1":      "0.4.post1",
+		"warning: noise\n0.4.1": "0.4.1",
+		"":                      Unknown,
+		"no numbers here":       Unknown,
+		"just 4123":             Unknown,
+	} {
+		if got := firstVersion(in); got != want {
+			t.Errorf("firstVersion(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func TestProbingIsBoundedByTheContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	bin := script(t, "python3", "sleep 60")
+	r := NewRegistry(Config{})
+	r.engines = map[string]Engine{"sglang": &SGLang{cfg: SGLangConfig{Python: bin}}}
+
+	// Registration waits on this. An agent that cannot reach the control plane
+	// is worse than one reporting an incomplete list.
+	if got := r.Probe(ctx); len(got) != 0 {
+		t.Errorf("a cancelled probe should report nothing, got %v", got)
+	}
+}

internal/agent/engines/sglang.go

@@ -0,0 +1,83 @@
+package engines
+
+import "strconv"
+
+// SGLangConfig locates the SGLang installation.
+type SGLangConfig struct {
+	// Python is the interpreter of the SGLang virtualenv. SGLang is launched as
+	// a module rather than a console script, which is the documented entry
+	// point and avoids depending on the script being on PATH.
+	Python string `koanf:"python"`
+}
+
+// SGLang builds SGLang launch commands.
+type SGLang struct{ cfg SGLangConfig }
+
+// Name identifies the engine in the catalog.
+func (*SGLang) Name() string { return "sglang" }
+
+// HealthPath is SGLang's readiness endpoint.
+func (*SGLang) HealthPath() string { return "/health" }
+
+// MetricsPath is SGLang's Prometheus endpoint.
+func (*SGLang) MetricsPath() string { return "/metrics" }
+
+// Validate checks the profile against the GPUs it would run on.
+func (*SGLang) Validate(p Profile, gpus []GPUInfo) error {
+	if err := validateCommon(p, gpus); err != nil {
+		return err
+	}
+	if q := quantizationOf(p.Args); q == "fp8" {
+		if err := requireComputeCapability(p, gpus, 8.9, "fp8 quantization"); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ServesHTTP reports whether this rank answers the API.
+//
+// Every SGLang rank runs the same launcher, but only node-rank 0 binds the HTTP
+// port; the others exist to contribute GPUs to the tensor-parallel group.
+func (*SGLang) ServesHTTP(s ShardSpec) bool { return s.Leader() }
+
+// Commands returns the processes for one shard.
+//
+// SGLang supports multi-node natively, so there is only ever one process: every
+// rank runs the same launcher with its own rank number and they rendezvous at
+// the leader's dist-init address.
+func (g *SGLang) Commands(s ShardSpec) ([]Command, error) {
+	c, err := g.serverCommand(s)
+	if err != nil {
+		return nil, err
+	}
+	return []Command{c}, nil
+}
+
+func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
+	python := g.cfg.Python
+	if python == "" {
+		python = "python3"
+	}
+	p := s.Profile
+
+	args := []string{"-m", "sglang.launch_server",
+		"--model-path", p.ModelPath,
+		"--host", "0.0.0.0",
+		"--port", strconv.Itoa(s.HTTPPort),
+		"--tp", strconv.Itoa(p.TP),
+	}
+	if p.ServedModelName != "" {
+		args = append(args, "--served-model-name", p.ServedModelName)
+	}
+	if s.NNodes > 1 {
+		args = append(args,
+			"--nnodes", strconv.Itoa(s.NNodes),
+			"--node-rank", strconv.Itoa(s.Rank),
+			"--dist-init-addr", s.LeaderAddr+":"+strconv.Itoa(s.LeaderPort),
+		)
+	}
+	args = append(args, p.Args...)
+
+	return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil
+}

internal/agent/engines/sim.go

@@ -0,0 +1,85 @@
+package engines
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+)
+
+// SimConfig locates the simulated engine.
+type SimConfig struct {
+	// Bin is the llmgw binary itself; the engine is one of its subcommands.
+	// Empty means whatever started this agent, which is right almost always.
+	Bin string `koanf:"bin"`
+}
+
+// Sim launches llmgw's own simulated engine.
+//
+// It is in the production registry on purpose. A cluster can be brought up and
+// exercised end to end — placement, fabrics, multi-node sequencing, failure and
+// rebuild — on machines that have no GPUs, which is the only way most of that
+// path gets tested at all. The multi-node launch sequence went four phases
+// without ever running, because the only things exercising it were stub
+// processes that exercised the command line and nothing else.
+//
+// Nothing about it pretends to be real: the model name it serves is whatever it
+// is told, the tokens are invented, and it says so in its own logs.
+type Sim struct{ cfg SimConfig }
+
+// Name identifies the engine in the catalog.
+func (*Sim) Name() string { return "sim" }
+
+// HealthPath is the simulated engine's readiness endpoint.
+func (*Sim) HealthPath() string { return "/health" }
+
+// MetricsPath is its Prometheus endpoint, in vLLM's vocabulary.
+func (*Sim) MetricsPath() string { return "/metrics" }
+
+// ServesHTTP reports whether this rank answers the API.
+//
+// Only the leader, exactly as vLLM and SGLang behave — which is the behaviour
+// this engine exists to let us test.
+func (*Sim) ServesHTTP(s ShardSpec) bool { return s.Leader() }
+
+// Validate accepts anything the arithmetic allows.
+//
+// No compute-capability checks and no quantization rules: there is no hardware
+// to be wrong about, and inventing refusals would make this a worse stand-in
+// for the engines it stands in for.
+func (*Sim) Validate(p Profile, gpus []GPUInfo) error {
+	return validateCommon(p, gpus)
+}
+
+// Commands returns the single process that makes up one shard.
+func (s *Sim) Commands(spec ShardSpec) ([]Command, error) {
+	bin := s.cfg.Bin
+	if bin == "" {
+		exe, err := os.Executable()
+		if err != nil {
+			return nil, fmt.Errorf("locating the llmgw binary for the sim engine: %w", err)
+		}
+		bin = exe
+	}
+	p := spec.Profile
+
+	args := []string{"sim-engine",
+		"--listen", "0.0.0.0:" + strconv.Itoa(spec.HTTPPort),
+		"--model", p.ServedModelName,
+	}
+	if spec.NNodes > 1 {
+		args = append(args,
+			"--nnodes", strconv.Itoa(spec.NNodes),
+			"--node-rank", strconv.Itoa(spec.Rank),
+			"--dist-init-addr", spec.LeaderAddr+":"+strconv.Itoa(spec.LeaderPort),
+		)
+	}
+	args = append(args, p.Args...)
+
+	return []Command{{
+		Name: "sim-engine", Path: bin, Args: args,
+		Env: materializeEnv(baseEnv(spec)),
+		// Workers dial the leader, so the leader has to be listening first.
+		// Both ranks express the same wait because both know the same address.
+		WaitForLeaderPort: spec.NNodes > 1 && !spec.Leader(),
+	}}, nil
+}

internal/agent/engines/sim_test.go

@@ -0,0 +1,88 @@
+package engines
+
+import (
+	"slices"
+	"testing"
+)
+
+func simSpec(rank, nnodes int) ShardSpec {
+	return ShardSpec{
+		ReplicaID: "rep-1", ShardID: "sh-1", Rank: rank, NNodes: nnodes,
+		LeaderAddr: "10.0.0.5", LeaderPort: 20001,
+		GPUIndices: []int{0, 1}, HTTPPort: 20000, MgmtInterface: "eth0",
+		HCAs: []string{"mlx5_0", "mlx5_1"},
+		Profile: Profile{
+			Name: "tp4", Engine: "sim", TP: 4, GPUsPerNode: 2, Nodes: nnodes,
+			ModelPath: "/models/x", ServedModelName: "big-model",
+		},
+	}
+}
+
+func TestOnlyTheLeaderServes(t *testing.T) {
+	// The behaviour that made replicas fail while they were coming up perfectly
+	// well: health checking a worker's port, which never opens.
+	if !(&Sim{}).ServesHTTP(simSpec(0, 2)) {
+		t.Error("the leader does not serve")
+	}
+	if (&Sim{}).ServesHTTP(simSpec(1, 2)) {
+		t.Error("a worker claims to serve")
+	}
+}
+
+func TestAMultiNodeShardCarriesItsRendezvous(t *testing.T) {
+	cmds, err := (&Sim{cfg: SimConfig{Bin: "/usr/local/bin/llmgw"}}).Commands(simSpec(1, 2))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(cmds) != 1 {
+		t.Fatalf("%d commands", len(cmds))
+	}
+	c := cmds[0]
+
+	if c.Args[0] != "sim-engine" {
+		t.Errorf("the engine is a subcommand of llmgw: %v", c.Args)
+	}
+	for flag, want := range map[string]string{
+		"--nnodes": "2", "--node-rank": "1", "--dist-init-addr": "10.0.0.5:20001",
+	} {
+		if got, _ := flagValue(c.Args, flag); got != want {
+			t.Errorf("%s = %q, want %q", flag, got, want)
+		}
+	}
+	// A worker must wait for the leader's port, or the launch is a race.
+	if !c.WaitForLeaderPort {
+		t.Error("a worker does not wait for the leader")
+	}
+}
+
+func TestTheLeaderDoesNotWaitForItself(t *testing.T) {
+	cmds, _ := (&Sim{cfg: SimConfig{Bin: "x"}}).Commands(simSpec(0, 2))
+	if cmds[0].WaitForLeaderPort {
+		t.Error("the leader waits for a port it is about to open")
+	}
+}
+
+func TestASingleNodeShardCarriesNoRendezvous(t *testing.T) {
+	cmds, _ := (&Sim{cfg: SimConfig{Bin: "x"}}).Commands(simSpec(0, 1))
+	for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {
+		if slices.Contains(cmds[0].Args, flag) {
+			t.Errorf("a single-node shard was given %s", flag)
+		}
+	}
+}
+
+func TestTheShardIsStillPinnedToItsFabric(t *testing.T) {
+	// The simulated engine does not use NCCL, but the environment is generated
+	// by the same code that generates it for the ones that do — so a cluster
+	// exercised with this engine is exercising that too.
+	cmds, _ := (&Sim{cfg: SimConfig{Bin: "x"}}).Commands(simSpec(1, 2))
+	env := cmds[0].Env
+	for _, want := range []string{
+		"NCCL_IB_HCA=mlx5_0,mlx5_1", "NCCL_SOCKET_IFNAME=eth0",
+		"CUDA_VISIBLE_DEVICES=0,1", "NCCL_IB_DISABLE=0",
+	} {
+		if !slices.Contains(env, want) {
+			t.Errorf("the environment is missing %s", want)
+		}
+	}
+}

internal/agent/engines/vllm.go

@@ -0,0 +1,149 @@
+package engines
+
+import (
+	"fmt"
+	"strconv"
+)
+
+// VLLMConfig locates the vLLM installation and picks how it is launched.
+type VLLMConfig struct {
+	// Bin is the vllm executable.
+	Bin string `koanf:"bin"`
+	// LaunchMode selects the multi-node mechanism: "ray" or "mp".
+	//
+	// Which one works depends on the vLLM version, and the flags have changed
+	// more than once (plan §14). Both paths exist so a cluster can pick the one
+	// its version supports instead of waiting for a code change.
+	LaunchMode string `koanf:"launch_mode"`
+	// RayBin is used when LaunchMode is "ray".
+	RayBin string `koanf:"ray_bin"`
+}
+
+// VLLM builds vLLM launch commands.
+type VLLM struct{ cfg VLLMConfig }
+
+// Name identifies the engine in the catalog.
+func (*VLLM) Name() string { return "vllm" }
+
+// HealthPath is vLLM's readiness endpoint.
+func (*VLLM) HealthPath() string { return "/health" }
+
+// MetricsPath is vLLM's Prometheus endpoint.
+func (*VLLM) MetricsPath() string { return "/metrics" }
+
+// Validate checks the profile against the GPUs it would run on.
+func (*VLLM) Validate(p Profile, gpus []GPUInfo) error {
+	if err := validateCommon(p, gpus); err != nil {
+		return err
+	}
+	// FP8 needs Hopper or Ada; on anything older vLLM either refuses or falls
+	// back silently to a slower path, and a silent fallback is worse.
+	if q := quantizationOf(p.Args); q == "fp8" {
+		if err := requireComputeCapability(p, gpus, 8.9, "fp8 quantization"); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ServesHTTP reports whether this rank answers the API.
+//
+// Under Ray a worker runs nothing but the Ray join, and under multiprocessing
+// only the driver binds the port. Either way it is the leader alone.
+func (*VLLM) ServesHTTP(s ShardSpec) bool { return s.Leader() }
+
+// Commands returns the processes for one shard, in start order.
+//
+// Single node, or multi-node under multiprocessing: one server process. Under
+// Ray: the leader starts a head and then the server, a worker starts only the
+// join. A Ray worker running no vLLM process at all is correct — the cluster
+// gives the leader its GPUs.
+func (v *VLLM) Commands(s ShardSpec) ([]Command, error) {
+	ray, useRay := v.rayCommand(s)
+	if useRay && !s.Leader() {
+		// A worker is the Ray join and nothing else. It blocks, so its exit is
+		// the shard's exit, which is what the supervisor needs.
+		return []Command{ray}, nil
+	}
+
+	server, err := v.serverCommand(s)
+	if err != nil {
+		return nil, err
+	}
+	if useRay {
+		return []Command{ray, server}, nil
+	}
+	return []Command{server}, nil
+}
+
+// serverCommand produces the vLLM server process.
+func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
+	bin := v.cfg.Bin
+	if bin == "" {
+		bin = "vllm"
+	}
+	p := s.Profile
+
+	args := []string{"serve", p.ModelPath,
+		"--host", "0.0.0.0",
+		"--port", strconv.Itoa(s.HTTPPort),
+		"--tensor-parallel-size", strconv.Itoa(p.TP),
+	}
+	if p.PP > 1 {
+		args = append(args, "--pipeline-parallel-size", strconv.Itoa(p.PP))
+	}
+	if p.ServedModelName != "" {
+		args = append(args, "--served-model-name", p.ServedModelName)
+	}
+
+	env := baseEnv(s)
+
+	if s.NNodes > 1 {
+		switch v.cfg.LaunchMode {
+		case "", "ray":
+			// Ray is started separately by the supervisor; vLLM then finds the
+			// cluster through the address in the environment.
+			env["VLLM_HOST_IP"] = s.LeaderAddr
+			env["RAY_ADDRESS"] = fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort)
+			args = append(args, "--distributed-executor-backend", "ray")
+		case "mp":
+			env["VLLM_HOST_IP"] = s.LeaderAddr
+			env["VLLM_PORT"] = strconv.Itoa(s.LeaderPort)
+			args = append(args, "--distributed-executor-backend", "mp")
+		default:
+			return Command{}, fmt.Errorf("vllm: unknown launch_mode %q (want \"ray\" or \"mp\")", v.cfg.LaunchMode)
+		}
+	}
+
+	// Profile arguments last, so an operator can override anything above.
+	args = append(args, p.Args...)
+
+	return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil
+}
+
+// rayCommand returns the command that starts or joins the Ray cluster, or
+// ok=false when the launch mode does not use Ray.
+func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) {
+	if s.NNodes <= 1 || (v.cfg.LaunchMode != "" && v.cfg.LaunchMode != "ray") {
+		return Command{}, false
+	}
+	bin := v.cfg.RayBin
+	if bin == "" {
+		bin = "ray"
+	}
+
+	args := []string{"start", "--block"}
+	if s.Leader() {
+		args = append(args, "--head", "--port", strconv.Itoa(s.LeaderPort))
+	} else {
+		args = append(args, "--address", fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort))
+	}
+	args = append(args, "--num-gpus", strconv.Itoa(len(s.GPUIndices)))
+
+	return Command{
+		Name: "ray", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
+		// The head must be listening before anything else happens: a worker
+		// cannot join, and the leader's server cannot claim GPUs, until it is.
+		WaitForLeaderPort: true,
+	}, true
+}

internal/agent/failover_test.go

@@ -0,0 +1,107 @@
+package agent
+
+import (
+	"context"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
+)
+
+type standby bool
+
+func (s standby) IsLeader() bool { return !bool(s) }
+
+// startPlain runs an agent service over h2c, which is enough to test who
+// accepts a connection and who refuses it.
+//
+// A nil leader means none was configured, which is every deployment that
+// existed before leadership did and has to keep accepting agents.
+func startPlain(t *testing.T, leader agentsvc.Leader) (*inventory.Store, string) {
+	t.Helper()
+	inv := inventory.NewStore()
+	svc := agentsvc.New(inv, newRecorder(), false, slog.New(slog.DiscardHandler))
+	if leader != nil {
+		svc.SetLeadership(leader)
+	}
+
+	path, handler := svc.Handler()
+	mux := http.NewServeMux()
+	mux.Handle(path, handler)
+	// h2c, because the agent speaks HTTP/2 with prior knowledge and a plain
+	// server negotiates HTTP/1.1 — which turns a clear refusal into a frame
+	// parsing error that says nothing about why.
+	srv := httptest.NewServer(h2c.NewHandler(mux, &http2.Server{}))
+	t.Cleanup(srv.Close)
+	return inv, srv.URL
+}
+
+func TestAStandbyRefusesAnAgent(t *testing.T) {
+	// Commands travel down the stream the agent opened, so whichever control
+	// plane holds it is the only one that can start anything on that node. A
+	// standby holding it means the leader's placements fail with "no agent
+	// connected" for a node that is plainly connected.
+	_, addr := startPlain(t, standby(true))
+
+	ag := runAgent(t, Config{NodeID: "n1", Server: addr, MgmtAddr: "127.0.0.1"})
+	err := ag.session(context.Background(), ag.cfg.Server)
+	if err == nil {
+		t.Fatal("a standby accepted an agent")
+	}
+	if !strings.Contains(err.Error(), "standby") {
+		t.Errorf("the refusal does not say why: %v", err)
+	}
+}
+
+func TestAnAgentReachesTheLeaderPastAStandby(t *testing.T) {
+	// The point of the list. The standby is first, so the agent only registers
+	// if it moved on rather than sitting on the refusal.
+	_, standbyAddr := startPlain(t, standby(true))
+	leaderInv, leaderAddr := startPlain(t, standby(false))
+
+	ag := runAgent(t, Config{
+		NodeID: "n1", MgmtAddr: "127.0.0.1",
+		Servers: []string{standbyAddr, leaderAddr},
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	go ag.Run(ctx)
+
+	deadline := time.Now().Add(8 * time.Second)
+	for time.Now().Before(deadline) {
+		if _, ok := leaderInv.Get("n1"); ok {
+			return
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	t.Fatal("the agent never reached the leader past the standby")
+}
+
+func TestALeaderStillAcceptsWithNoLeadershipConfigured(t *testing.T) {
+	// Every deployment before this had no leadership at all, and must keep
+	// accepting agents with nothing configured.
+	inv, addr := startPlain(t, nil)
+
+	ag := runAgent(t, Config{NodeID: "n1", Server: addr, MgmtAddr: "127.0.0.1"})
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	go ag.Run(ctx)
+
+	deadline := time.Now().Add(4 * time.Second)
+	for time.Now().Before(deadline) {
+		if _, ok := inv.Get("n1"); ok {
+			return
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	t.Fatal("a service with no leadership configured refused an agent")
+}

internal/agent/logring.go

@@ -0,0 +1,84 @@
+package agent
+
+import (
+	"context"
+	"log/slog"
+	"strings"
+	"sync"
+)
+
+// agentLogLines is how much of its own log an agent keeps for the admin page.
+// Enough to cover an engine install or a failed start; the journal keeps the
+// rest for anyone who can reach it.
+const agentLogLines = 2000
+
+// logRing keeps the most recent lines written to it.
+type logRing struct {
+	mu    sync.Mutex
+	lines []string
+	next  int
+	full  bool
+}
+
+func newLogRing(n int) *logRing { return &logRing{lines: make([]string, n)} }
+
+// Write takes whole lines, as a text handler writes them.
+func (r *logRing) Write(p []byte) (int, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for _, line := range strings.Split(strings.TrimRight(string(p), "\n"), "\n") {
+		r.lines[r.next] = line
+		r.next = (r.next + 1) % len(r.lines)
+		if r.next == 0 {
+			r.full = true
+		}
+	}
+	return len(p), nil
+}
+
+// tail returns up to n of the most recent lines, oldest first.
+func (r *logRing) tail(n int) []string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	size := r.next
+	if r.full {
+		size = len(r.lines)
+	}
+	n = min(n, size)
+	out := make([]string, 0, n)
+	for i := size - n; i < size; i++ {
+		idx := i
+		if r.full {
+			idx = (r.next + i) % len(r.lines)
+		}
+		out = append(out, r.lines[idx])
+	}
+	return out
+}
+
+// teeHandler sends every record to two handlers: wherever the agent already
+// logs, and the ring the admin page reads.
+type teeHandler struct{ a, b slog.Handler }
+
+func (t teeHandler) Enabled(ctx context.Context, l slog.Level) bool {
+	return t.a.Enabled(ctx, l) || t.b.Enabled(ctx, l)
+}
+
+func (t teeHandler) Handle(ctx context.Context, r slog.Record) error {
+	var err error
+	if t.a.Enabled(ctx, r.Level) {
+		err = t.a.Handle(ctx, r.Clone())
+	}
+	if t.b.Enabled(ctx, r.Level) {
+		_ = t.b.Handle(ctx, r.Clone())
+	}
+	return err
+}
+
+func (t teeHandler) WithAttrs(as []slog.Attr) slog.Handler {
+	return teeHandler{t.a.WithAttrs(as), t.b.WithAttrs(as)}
+}
+
+func (t teeHandler) WithGroup(name string) slog.Handler {
+	return teeHandler{t.a.WithGroup(name), t.b.WithGroup(name)}
+}

internal/agent/logring_test.go

@@ -0,0 +1,49 @@
+package agent
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+	"testing"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+)
+
+func TestTheLogRingKeepsTheNewestLinesInOrder(t *testing.T) {
+	r := newLogRing(3)
+	if got := r.tail(10); len(got) != 0 {
+		t.Fatalf("empty ring gave %v", got)
+	}
+	for i := 1; i <= 5; i++ {
+		fmt.Fprintf(r, "line %d\n", i)
+	}
+	if got := r.tail(10); !slices.Equal(got, []string{"line 3", "line 4", "line 5"}) {
+		t.Errorf("after wrapping: %v", got)
+	}
+	if got := r.tail(2); !slices.Equal(got, []string{"line 4", "line 5"}) {
+		t.Errorf("tail(2): %v", got)
+	}
+}
+
+func TestANodeAnswersForItsOwnLog(t *testing.T) {
+	cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(), EnginesDir: t.TempDir()}
+	cfg.Superv.WorkDir = t.TempDir()
+	a, err := New(cfg, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.log.Info("engine installation", "state", "installing")
+
+	r := a.logsReply(&llmgwv1.LogsRequest{RequestId: "x", Source: "agent", Lines: 50})
+	if r.GetRequestId() != "x" || !strings.Contains(strings.Join(r.GetLines(), "\n"), "engine installation") {
+		t.Errorf("the agent's own line is not in its log: %v", r.GetLines())
+	}
+	if !slices.Contains(r.GetSources(), "agent") {
+		t.Errorf("sources %v", r.GetSources())
+	}
+
+	r = a.logsReply(&llmgwv1.LogsRequest{RequestId: "y", Source: "no-such-shard"})
+	if r.GetError() == "" {
+		t.Error("an unknown engine was answered as if it existed")
+	}
+}

internal/agent/mtls_test.go

@@ -0,0 +1,317 @@
+package agent
+
+import (
+	"context"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
+)
+
+// pki issues a cluster's certificates into a temporary directory and returns
+// the configurations the control plane and one agent would load.
+func pki(t *testing.T, nodeID string) (server, client tlsconf.Config) {
+	t.Helper()
+	dir := t.TempDir()
+
+	ca, err := tlsconf.NewCA("test CA")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := ca.Write(dir); err != nil {
+		t.Fatal(err)
+	}
+	srv, err := ca.IssueServer("control-plane", []string{"127.0.0.1"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := srv.Write(dir, "server"); err != nil {
+		t.Fatal(err)
+	}
+	cli, err := ca.IssueClient(nodeID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := cli.Write(dir, "agent"); err != nil {
+		t.Fatal(err)
+	}
+
+	caPath := filepath.Join(dir, "ca.pem")
+	return tlsconf.Config{CA: caPath, Cert: filepath.Join(dir, "server.pem"), Key: filepath.Join(dir, "server-key.pem")},
+		tlsconf.Config{CA: caPath, Cert: filepath.Join(dir, "agent.pem"), Key: filepath.Join(dir, "agent-key.pem")}
+}
+
+// startTLS runs a real service behind a real TLS listener, with identity
+// required — the deployed configuration rather than an approximation of it.
+func startTLS(t *testing.T, serverTLS tlsconf.Config) (*agentsvc.Service, *inventory.Store, string) {
+	t.Helper()
+
+	inv := inventory.NewStore()
+	svc := agentsvc.New(inv, newRecorder(), true, slog.New(slog.DiscardHandler))
+	path, handler := svc.Handler()
+	mux := http.NewServeMux()
+	mux.Handle(path, handler)
+
+	tc, err := serverTLS.ServerConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	srv := httptest.NewUnstartedServer(mux)
+	srv.Listener = raw
+	srv.TLS = tc
+	srv.StartTLS()
+	t.Cleanup(srv.Close)
+
+	return svc, inv, "https://" + raw.Addr().String()
+}
+
+func runAgent(t *testing.T, cfg Config) *Agent {
+	t.Helper()
+
+	cfg.MgmtAddr = "127.0.0.1"
+	cfg.HeartbeatInterval = 50 * time.Millisecond
+	cfg.ReconnectBase = 20 * time.Millisecond
+	cfg.ReconnectMax = 100 * time.Millisecond
+	cfg.Discovery.SkipGPUs = true
+	cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent")
+	cfg.Superv = supervisor.Config{
+		WorkDir: t.TempDir(), LogLines: 64,
+		StopTimeout: 2 * time.Second, HealthInterval: 20 * time.Millisecond,
+	}
+
+	client, err := Client(cfg.TLS, 5*time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ag, err := New(cfg, client, slog.New(slog.DiscardHandler))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() { _ = ag.Run(ctx) }()
+	t.Cleanup(func() {
+		cancel()
+		ag.Supervisor().StopAll(context.Background())
+	})
+	return ag
+}
+
+func registered(svc *agentsvc.Service, inv *inventory.Store, nodeID string) func() bool {
+	return func() bool {
+		_, ok := inv.Get(nodeID)
+		return ok && len(svc.ConnectedNodes()) == 1
+	}
+}
+
+func eventually(t *testing.T, limit time.Duration, pred func() bool) bool {
+	t.Helper()
+	deadline := time.Now().Add(limit)
+	for time.Now().Before(deadline) {
+		if pred() {
+			return true
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	return false
+}
+
+func TestAgentRegistersOverMutualTLS(t *testing.T) {
+	serverTLS, clientTLS := pki(t, "llmgw-node01")
+	svc, inv, url := startTLS(t, serverTLS)
+
+	cfg := DefaultConfig()
+	cfg.NodeID = "llmgw-node01"
+	cfg.Server = url
+	cfg.TLS = clientTLS
+	runAgent(t, cfg)
+
+	if !eventually(t, 10*time.Second, registered(svc, inv, "llmgw-node01")) {
+		t.Fatal("the agent never registered over mutual TLS")
+	}
+}
+
+func TestNodeCannotClaimAnotherNodesIdentity(t *testing.T) {
+	// The point of the whole exercise. node01 holds a valid certificate — it is
+	// a legitimate member of the cluster — and still cannot register as node02,
+	// because the id it claims is checked against the one it can prove.
+	serverTLS, clientTLS := pki(t, "llmgw-node01")
+	svc, inv, url := startTLS(t, serverTLS)
+
+	cfg := DefaultConfig()
+	cfg.NodeID = "llmgw-node02"
+	cfg.Server = url
+	cfg.TLS = clientTLS
+	runAgent(t, cfg)
+
+	// The agent reconnects on failure, so this has to stay false throughout
+	// rather than merely start false.
+	if eventually(t, 2*time.Second, func() bool {
+		_, ok := inv.Get("llmgw-node02")
+		return ok || len(svc.ConnectedNodes()) > 0
+	}) {
+		t.Fatal("an agent registered under a node id its certificate does not authorize")
+	}
+}
+
+func TestUnauthenticatedAgentIsRefused(t *testing.T) {
+	// The defensive half of the check: identity is required, but the connection
+	// carried none. In the shipped wiring TLS and this flag are tied together,
+	// so reaching here means someone untied them — and the answer must still be
+	// refusal rather than a silently trusted node id.
+	inv := inventory.NewStore()
+	svc := agentsvc.New(inv, newRecorder(), true, slog.New(slog.DiscardHandler))
+	path, handler := svc.Handler()
+	mux := http.NewServeMux()
+	mux.Handle(path, handler)
+
+	srv := httptest.NewServer(h2c.NewHandler(mux, &http2.Server{}))
+	t.Cleanup(srv.Close)
+
+	cfg := DefaultConfig()
+	cfg.NodeID = "llmgw-node01"
+	cfg.Server = srv.URL
+	runAgent(t, cfg)
+
+	if eventually(t, 2*time.Second, registered(svc, inv, "llmgw-node01")) {
+		t.Fatal("an agent with no proven identity reached the control plane")
+	}
+}
+
+func TestPlaintextAgentIsRejectedWhenTLSIsConfigured(t *testing.T) {
+	// A misconfiguration worth catching at startup rather than at runtime: an
+	// agent holding certificates but dialling http:// would connect in the
+	// clear and never learn that it had.
+	_, clientTLS := pki(t, "llmgw-node01")
+
+	cfg := DefaultConfig()
+	cfg.NodeID = "llmgw-node01"
+	cfg.Server = "http://127.0.0.1:9443"
+	cfg.TLS = clientTLS
+
+	if _, err := New(cfg, nil, slog.New(slog.DiscardHandler)); err == nil {
+		t.Fatal("an agent with certificates was allowed to dial a plaintext address")
+	}
+}
+
+func TestBareAddressGainsTheRightScheme(t *testing.T) {
+	_, clientTLS := pki(t, "llmgw-node01")
+
+	secure := Config{NodeID: "n", Server: "10.0.0.140:9443", TLS: clientTLS}
+	if err := secure.normalize(); err != nil {
+		t.Fatal(err)
+	}
+	if secure.Server != "https://10.0.0.140:9443" {
+		t.Errorf("a configured agent dials %q", secure.Server)
+	}
+
+	plain := Config{NodeID: "n", Server: "10.0.0.140:9443"}
+	if err := plain.normalize(); err != nil {
+		t.Fatal(err)
+	}
+	if plain.Server != "http://10.0.0.140:9443" {
+		t.Errorf("an unconfigured agent dials %q", plain.Server)
+	}
+}
+
+func TestRefusedAgentLearnsWhy(t *testing.T) {
+	// An agent that cannot register must be able to say why. Reporting a
+	// refusal as a clean disconnect leaves an operator with a silent reconnect
+	// loop and nothing to act on — and resets the backoff, so the loop is a
+	// fast one.
+	serverTLS, clientTLS := pki(t, "llmgw-node01")
+	_, _, url := startTLS(t, serverTLS)
+
+	cfg := DefaultConfig()
+	cfg.NodeID = "llmgw-node02"
+	cfg.Server = url
+	cfg.TLS = clientTLS
+	cfg.Discovery.SkipGPUs = true
+	cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent")
+	cfg.Superv = supervisor.Config{WorkDir: t.TempDir(), LogLines: 8}
+
+	client, err := Client(cfg.TLS, 5*time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ag, err := New(cfg, client, slog.New(slog.DiscardHandler))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = ag.session(context.Background(), ag.cfg.Server)
+	if err == nil {
+		t.Fatal("a refused session reported success")
+	}
+	if !strings.Contains(err.Error(), "llmgw-node01") || !strings.Contains(err.Error(), "llmgw-node02") {
+		t.Errorf("the error names neither certificate nor claim: %v", err)
+	}
+}
+
+// TestAnAgentWithNothingRunningStillTellsTheControlPlane covers the gap that
+// made a reconciler fix dead code in production.
+//
+// "I am running nothing" is information: it is how the control plane learns
+// that an engine died while its agent survived. The service used to return
+// early on an empty list, so that message never reached the reconciler — while
+// the reconciler's own tests, which call it directly, kept passing.
+//
+// This test goes through the real stream for that reason.
+func TestAnAgentWithNothingRunningStillTellsTheControlPlane(t *testing.T) {
+	serverTLS, clientTLS := pki(t, "llmgw-node01")
+
+	inv := inventory.NewStore()
+	rec := newRecorder()
+	svc := agentsvc.New(inv, rec, true, slog.New(slog.DiscardHandler))
+	path, handler := svc.Handler()
+	mux := http.NewServeMux()
+	mux.Handle(path, handler)
+
+	tc, err := serverTLS.ServerConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	srv := httptest.NewUnstartedServer(mux)
+	srv.Listener = raw
+	srv.TLS = tc
+	srv.StartTLS()
+	t.Cleanup(srv.Close)
+
+	cfg := DefaultConfig()
+	cfg.NodeID = "llmgw-node01"
+	cfg.Server = "https://" + raw.Addr().String()
+	cfg.TLS = clientTLS
+	runAgent(t, cfg)
+
+	// The agent has no shards at all, which is the whole point.
+	if !eventually(t, 10*time.Second, func() bool {
+		return len(rec.adoptCalls()) > 0
+	}) {
+		t.Fatal("an agent running nothing never reached the reconciler")
+	}
+	for _, call := range rec.adoptCalls() {
+		if call.nodeID != "llmgw-node01" {
+			t.Errorf("adopt was told node %q", call.nodeID)
+		}
+	}
+}

internal/agent/provision/install.go

@@ -0,0 +1,658 @@
+package provision
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"log/slog"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+)
+
+// Want is one engine at one version.
+type Want struct {
+	Engine  string
+	Version string
+}
+
+// The states an engine's installation can be in.
+const (
+	Installing  = "installing"
+	Ready       = "ready"
+	Failed      = "failed"
+	Unsupported = "unsupported"
+)
+
+// Status is where one engine's installation stands.
+type Status struct {
+	Engine  string
+	Version string
+	Variant string
+	State   string
+	// Detail is for a person: what is happening, or what went wrong.
+	Detail string
+	// Binary is the executable, once ready.
+	Binary string
+}
+
+// Installer installs engines under one directory.
+type Installer struct {
+	// Root is where engines are installed, one directory per build:
+	// <root>/<engine>/<version>-<variant>.
+	Root string
+	Host discovery.Host
+	Log  *slog.Logger
+	HTTP *http.Client
+	// OnStatus hears every change, so a download that takes minutes is seen
+	// to be under way rather than silent.
+	OnStatus func(Status)
+
+	// Run executes a command; a seam for tests. Nil runs it for real.
+	Run func(ctx context.Context, env []string, name string, args ...string) ([]byte, error)
+	// APIBase and DownloadBase default to GitHub; tests point them at a fake.
+	APIBase      string
+	DownloadBase string
+}
+
+// DefaultRoot is where engines are installed when nothing says otherwise.
+const DefaultRoot = "/opt/llmgw/engines"
+
+// marker is written into a build's directory once it has been installed and
+// seen to run. Its absence means the directory is not to be trusted.
+const marker = ".llmgw-installed"
+
+type markerFile struct {
+	Engine    string            `json:"engine"`
+	Version   string            `json:"version"`
+	Variant   string            `json:"variant"`
+	Binary    string            `json:"binary"`
+	SHA256    map[string]string `json:"sha256"`
+	Installed time.Time         `json:"installed"`
+}
+
+// Converge brings every wanted engine to ready, or says why it cannot.
+//
+// An engine already installed is checked again rather than believed: its
+// directory can be damaged, or a library it needs removed by somebody's
+// upgrade, and that is found here rather than by the next replica to fail.
+func (in *Installer) Converge(ctx context.Context, wants []Want) []Status {
+	out := make([]Status, 0, len(wants))
+	for _, w := range wants {
+		st := in.one(ctx, w)
+		in.report(st)
+		out = append(out, st)
+	}
+	return out
+}
+
+func (in *Installer) one(ctx context.Context, w Want) Status {
+	st := Status{Engine: w.Engine, Version: w.Version}
+	recipe, ok := Recipes[w.Engine]
+	if !ok {
+		st.State = Unsupported
+		st.Detail = "this agent does not know how to install " + w.Engine +
+			"; install it by hand and name it in the agent's configuration"
+		return st
+	}
+	build, err := recipe.Choose(w.Version, in.Host)
+	if err != nil {
+		st.State, st.Detail = Unsupported, err.Error()
+		return st
+	}
+	st.Variant = build.Variant
+	dir := filepath.Join(in.root(), w.Engine, w.Version+"-"+build.Variant)
+
+	// Installed before: check it still runs, and trust nothing else about it.
+	if _, err := os.Stat(filepath.Join(dir, marker)); err == nil {
+		// The packages are checked too. Removing one is exactly the kind of
+		// change an unrelated upgrade makes, and ldd below would catch it only
+		// as a missing library with no hint of which package held it.
+		if err := in.packages(ctx, build.Packages, &st); err != nil {
+			st.State, st.Detail = Failed, err.Error()
+			return st
+		}
+		bin, version, err := in.verify(ctx, dir, build)
+		if err == nil {
+			return in.ready(st, build, bin, version)
+		}
+		in.log().Warn("an installed engine no longer runs; installing it again",
+			"engine", w.Engine, "version", w.Version, "err", err)
+	}
+
+	if err := in.install(ctx, recipe, w, build, dir, &st); err != nil {
+		st.State, st.Detail, st.Binary = Failed, err.Error(), ""
+		return st
+	}
+	bin, version, err := in.verify(ctx, dir, build)
+	if err != nil {
+		st.State, st.Detail = Failed, err.Error()
+		return st
+	}
+	return in.ready(st, build, bin, version)
+}
+
+func (in *Installer) ready(st Status, b Build, bin, version string) Status {
+	st.State, st.Binary = Ready, bin
+	st.Detail = strings.TrimSpace(filepath.Base(bin) + " " + version)
+	var missing []string
+	for _, f := range b.CPUFlags {
+		if !in.Host.HasCPUFlag(f) {
+			missing = append(missing, f)
+		}
+	}
+	if len(missing) > 0 {
+		// Said on every report, not once in a log: it is the difference between
+		// five tokens a second and forty, and a VM given QEMU's default CPU
+		// model looks entirely healthy otherwise.
+		st.Detail += "; this CPU lacks " + strings.Join(missing, ", ") +
+			", so it will run several times slower (a VM needs CPU type \"host\")"
+	}
+	return st
+}
+
+// install fetches, checks and unpacks a build into dir.
+//
+// Into a scratch directory renamed into place at the end, so a failure halfway
+// leaves nothing that looks installed.
+func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error {
+	if err := in.packages(ctx, b.Packages, st); err != nil {
+		return err
+	}
+
+	in.progress(st, "asking GitHub for the checksums of %s %s", w.Engine, w.Version)
+	sums, err := in.checksums(ctx, r.Repo, w.Version)
+	if err != nil {
+		return err
+	}
+
+	tmp := dir + ".partial"
+	_ = os.RemoveAll(tmp)
+	if err := os.MkdirAll(tmp, 0o755); err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmp) //nolint:errcheck // gone either way once renamed
+
+	got := map[string]string{}
+	for _, asset := range b.Assets {
+		want, ok := sums[asset]
+		if !ok {
+			// Refused rather than installed unchecked: this is a binary that
+			// will run as root on every node.
+			return fmt.Errorf("%s %s publishes no checksum for %s, so it cannot be verified",
+				w.Engine, w.Version, asset)
+		}
+		archive := filepath.Join(tmp, asset+".download")
+		if err := in.download(ctx, r.Repo, w.Version, asset, archive, want, st); err != nil {
+			return err
+		}
+		in.progress(st, "unpacking %s", asset)
+		if err := untarGz(archive, tmp); err != nil {
+			return fmt.Errorf("unpacking %s: %w", asset, err)
+		}
+		_ = os.Remove(archive)
+		got[asset] = want
+	}
+
+	in.progress(st, "checking that %s runs", b.Binary)
+	bin, _, err := in.verify(ctx, tmp, b)
+	if err != nil {
+		return err
+	}
+	rel, _ := filepath.Rel(tmp, bin)
+	m, _ := json.MarshalIndent(markerFile{
+		Engine: w.Engine, Version: w.Version, Variant: b.Variant,
+		Binary: rel, SHA256: got, Installed: time.Now().UTC(),
+	}, "", "  ")
+	if err := os.WriteFile(filepath.Join(tmp, marker), m, 0o644); err != nil { //nolint:gosec // not secret
+		return err
+	}
+	_ = os.RemoveAll(dir)
+	if err := os.Rename(tmp, dir); err != nil {
+		return err
+	}
+	in.log().Info("installed an engine", "engine", w.Engine, "version", w.Version,
+		"variant", b.Variant, "dir", dir)
+	return nil
+}
+
+// verify finds the binary and runs it.
+//
+// Two checks, because they fail differently. ldd names a missing library
+// outright, where running the binary would only say it could not start. And
+// running it is the only proof that the rest — the right instruction set, the
+// right driver — is in place.
+func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, version string, err error) {
+	bin, err = findFile(dir, b.Binary)
+	if err != nil {
+		return "", "", err
+	}
+	if out, err := in.run(ctx, nil, "ldd", bin); err == nil {
+		if missing := lddMissing(string(out)); len(missing) > 0 {
+			return "", "", fmt.Errorf("%s cannot run on this system: %s", b.Binary, strings.Join(missing, "; "))
+		}
+	}
+	tctx, cancel := context.WithTimeout(ctx, 60*time.Second)
+	defer cancel()
+	out, err := in.run(tctx, nil, bin, "--version")
+	if err != nil {
+		return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out))
+	}
+	return bin, firstVersionLine(out), nil
+}
+
+// packages installs the operating system packages a build needs.
+//
+// Only those missing, and from the node's own package sources: a package the
+// system already has is not touched, so an engine never upgrades something
+// the rest of the machine depends on.
+func (in *Installer) packages(ctx context.Context, byFamily map[string][]string, st *Status) error {
+	family := in.Host.Family()
+	pkgs := byFamily[family]
+	if len(pkgs) == 0 {
+		// A system of no family this knows is not refused: its packages may
+		// well be there, and if they are not, ldd names the missing library.
+		return nil
+	}
+	pm := packageManagers[family]
+	var missing []string
+	for _, p := range pkgs {
+		if _, err := in.run(ctx, nil, pm.query[0], append(pm.query[1:], p)...); err != nil {
+			missing = append(missing, p)
+		}
+	}
+	if len(missing) == 0 {
+		return nil
+	}
+	tool, cmds := pm.install(in, missing)
+	if tool == "" {
+		return fmt.Errorf("%s is missing and no package manager was found to install it",
+			strings.Join(missing, ", "))
+	}
+	in.progress(st, "installing %s with %s", strings.Join(missing, ", "), tool)
+	for _, c := range cmds {
+		if err := in.runWaitingForLock(ctx, pm.env, c, st); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// lockWait bounds how long a package command waits for another to finish.
+const lockWait = 10 * time.Minute
+
+// lockRetry is how often a locked package manager is tried again.
+var lockRetry = 10 * time.Second
+
+// runWaitingForLock runs a package command, waiting while another holds the
+// package manager's lock.
+//
+// A machine a minute old is running cloud-init's and unattended-upgrades' own
+// apt. apt's DPkg::Lock::Timeout covers dpkg's lock and not the one `apt-get
+// update` takes on its lists, so the first install on a new node failed with
+// "Unable to lock directory /var/lib/apt/lists/" — found by building one.
+func (in *Installer) runWaitingForLock(ctx context.Context, env, c []string, st *Status) error {
+	deadline := time.Now().Add(lockWait)
+	for {
+		out, err := in.run(ctx, env, c[0], c[1:]...)
+		if err == nil {
+			return nil
+		}
+		if !lockHeld(string(out)) || time.Now().After(deadline) {
+			return fmt.Errorf("%s: %v: %s", strings.Join(c, " "), err, lastLine(out))
+		}
+		in.progress(st, "waiting for another %s to finish (a new machine runs its own updates first)", c[0])
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(lockRetry):
+		}
+	}
+}
+
+// lockHeld reports whether a package command failed only because another
+// held the lock.
+func lockHeld(out string) bool {
+	for _, s := range []string{"Unable to lock", "Could not get lock", "is locked by another process",
+		"waiting for process with pid", "System management is locked"} {
+		if strings.Contains(out, s) {
+			return true
+		}
+	}
+	return false
+}
+
+// packageManager is how one packaging family asks and installs.
+type packageManager struct {
+	// query exits zero when the package named after it is installed.
+	query []string
+	env   []string
+	// install picks the tool present on this machine and the commands to run.
+	install func(in *Installer, pkgs []string) (tool string, cmds [][]string)
+}
+
+var packageManagers = map[string]packageManager{
+	"deb": {
+		// dpkg -s exits zero for a package merely known about, so the status
+		// is asked for and checked by the command itself: ${db:Status-Abbrev}
+		// is "ii " only when installed.
+		query: []string{"sh", "-c", `test "$(dpkg-query -W -f='${db:Status-Abbrev}' "$0" 2>/dev/null)" = "ii "`},
+		env:   []string{"DEBIAN_FRONTEND=noninteractive"},
+		install: func(_ *Installer, pkgs []string) (string, [][]string) {
+			// A lock timeout rather than failing on the lock: a machine a
+			// minute old is running cloud-init's and unattended-upgrades' own
+			// apt, and the first attempt on every new node would fail.
+			lock := "-o=DPkg::Lock::Timeout=600"
+			return "apt-get", [][]string{
+				{"apt-get", lock, "update"},
+				append([]string{"apt-get", lock, "install", "-y", "--no-install-recommends"}, pkgs...),
+			}
+		},
+	},
+	"rpm": {
+		query: []string{"rpm", "-q"},
+		install: func(in *Installer, pkgs []string) (string, [][]string) {
+			for _, tool := range []string{"dnf", "yum"} {
+				if in.has(tool) {
+					return tool, [][]string{append([]string{tool, "install", "-y"}, pkgs...)}
+				}
+			}
+			if in.has("zypper") {
+				return "zypper", [][]string{append([]string{"zypper", "--non-interactive", "install"}, pkgs...)}
+			}
+			return "", nil
+		},
+	},
+}
+
+// has reports whether a command is on PATH. Asked through the same seam as
+// every other command, so a test can say which tools a machine has.
+func (in *Installer) has(tool string) bool {
+	_, err := in.run(context.Background(), nil, "sh", "-c", `command -v "$0"`, tool)
+	return err == nil
+}
+
+// checksums reads the sha256 GitHub publishes for each asset of a release.
+func (in *Installer) checksums(ctx context.Context, repo, tag string) (map[string]string, error) {
+	url := strings.TrimSuffix(in.apiBase(), "/") + "/repos/" + repo + "/releases/tags/" + tag
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept", "application/vnd.github+json")
+	resp, err := in.client().Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("reaching GitHub for %s %s: %w", repo, tag, err)
+	}
+	defer resp.Body.Close()
+	switch {
+	case resp.StatusCode == http.StatusNotFound:
+		return nil, fmt.Errorf("%s has no release %s", repo, tag)
+	case resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests:
+		// Unauthenticated callers get sixty requests an hour per address, and
+		// a whole fleet behind one NAT shares them.
+		return nil, fmt.Errorf("GitHub is rate-limiting this address (%s); it retries later", resp.Status)
+	case resp.StatusCode != http.StatusOK:
+		return nil, fmt.Errorf("GitHub answered %s for %s %s", resp.Status, repo, tag)
+	}
+	var rel struct {
+		Assets []struct {
+			Name   string `json:"name"`
+			Digest string `json:"digest"`
+		} `json:"assets"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&rel); err != nil {
+		return nil, fmt.Errorf("reading %s %s from GitHub: %w", repo, tag, err)
+	}
+	out := map[string]string{}
+	for _, a := range rel.Assets {
+		if sum, ok := strings.CutPrefix(a.Digest, "sha256:"); ok && len(sum) == 64 {
+			out[a.Name] = sum
+		}
+	}
+	return out, nil
+}
+
+func (in *Installer) download(ctx context.Context, repo, tag, asset, dst, want string, st *Status) error {
+	url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + repo + "/releases/download/" + tag + "/" + asset
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := in.client().Do(req)
+	if err != nil {
+		return fmt.Errorf("downloading %s: %w", asset, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("downloading %s: %s", asset, resp.Status)
+	}
+	if resp.ContentLength > 0 {
+		in.progress(st, "downloading %s (%d MB)", asset, resp.ContentLength>>20)
+	} else {
+		in.progress(st, "downloading %s", asset)
+	}
+	f, err := os.Create(dst) //nolint:gosec // inside the install root
+	if err != nil {
+		return err
+	}
+	h := sha256.New()
+	_, err = io.Copy(io.MultiWriter(f, h), resp.Body)
+	if cerr := f.Close(); err == nil {
+		err = cerr
+	}
+	if err != nil {
+		return fmt.Errorf("downloading %s: %w", asset, err)
+	}
+	if got := hex.EncodeToString(h.Sum(nil)); got != want {
+		return fmt.Errorf("%s does not match the checksum its publisher gave (got %s, expected %s)",
+			asset, got[:12], want[:12])
+	}
+	return nil
+}
+
+// untarGz unpacks an archive under dir, refusing anything that would land
+// outside it. A release archive is somebody else's file, and a path of
+// "../../etc/cron.d/x" in it would otherwise be written as root.
+func untarGz(archive, dir string) error {
+	f, err := os.Open(archive) //nolint:gosec // inside the install root
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	gz, err := gzip.NewReader(f)
+	if err != nil {
+		return err
+	}
+	tr := tar.NewReader(gz)
+	for {
+		h, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		target, err := within(dir, h.Name)
+		if err != nil {
+			return err
+		}
+		switch h.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(target, 0o755); err != nil {
+				return err
+			}
+		case tar.TypeReg:
+			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+				return err
+			}
+			out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755) //nolint:gosec // checked above
+			if err != nil {
+				return err
+			}
+			if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // a release archive, size bounded by its checksum
+				out.Close()
+				return err
+			}
+			if err := out.Close(); err != nil {
+				return err
+			}
+		case tar.TypeSymlink:
+			// Libraries are versioned through symlinks, and those are
+			// relative. One pointing anywhere else is refused.
+			if filepath.IsAbs(h.Linkname) {
+				return fmt.Errorf("%s links outside the archive", h.Name)
+			}
+			if _, err := within(dir, filepath.Join(filepath.Dir(h.Name), h.Linkname)); err != nil {
+				return fmt.Errorf("%s links outside the archive", h.Name)
+			}
+			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+				return err
+			}
+			_ = os.Remove(target)
+			if err := os.Symlink(h.Linkname, target); err != nil {
+				return err
+			}
+		default:
+			// Devices, FIFOs and hard links have no business in a release.
+		}
+	}
+}
+
+func within(dir, name string) (string, error) {
+	target := filepath.Join(dir, name) //nolint:gosec // checked on the next line
+	rel, err := filepath.Rel(dir, target)
+	if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
+		return "", fmt.Errorf("%s would unpack outside the install directory", name)
+	}
+	return target, nil
+}
+
+// lddMissing reads what ldd could not satisfy, in two shapes:
+//
+//	libgomp.so.1 => not found
+//	/lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by ...)
+//
+// The second is a system too old for the build — RHEL 9 against a build that
+// wants GCC 12's libstdc++ — and is said as such, rather than as a library
+// that is plainly there being "not found".
+func lddMissing(out string) []string {
+	seen := map[string]bool{}
+	var missing []string
+	for _, line := range strings.Split(out, "\n") {
+		line = strings.TrimSpace(line)
+		if !strings.Contains(line, "not found") {
+			continue
+		}
+		var what string
+		if lib, rest, ok := strings.Cut(line, ": version `"); ok {
+			sym, _, _ := strings.Cut(rest, "'")
+			what = filepath.Base(lib) + " is too old (needs " + sym + ")"
+		} else {
+			what = strings.TrimSpace(strings.SplitN(line, "=>", 2)[0]) + " is not installed"
+		}
+		if !seen[what] {
+			seen[what] = true
+			missing = append(missing, what)
+		}
+	}
+	return missing
+}
+
+func findFile(dir, name string) (string, error) {
+	var found string
+	_ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
+		if err != nil || found != "" {
+			return nil //nolint:nilerr // an unreadable subtree is not the binary
+		}
+		if !d.IsDir() && d.Name() == name {
+			found = p
+		}
+		return nil
+	})
+	if found == "" {
+		return "", fmt.Errorf("the build has no %s in it", name)
+	}
+	return found, nil
+}
+
+func (in *Installer) progress(st *Status, format string, args ...any) {
+	st.State, st.Detail = Installing, fmt.Sprintf(format, args...)
+	in.report(*st)
+}
+
+func (in *Installer) report(st Status) {
+	if in.OnStatus != nil {
+		in.OnStatus(st)
+	}
+}
+
+func (in *Installer) run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
+	if in.Run != nil {
+		return in.Run(ctx, env, name, args...)
+	}
+	cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // fixed commands and paths we installed
+	cmd.Env = append(os.Environ(), env...)
+	return cmd.CombinedOutput()
+}
+
+func (in *Installer) root() string {
+	if in.Root != "" {
+		return in.Root
+	}
+	return DefaultRoot
+}
+
+func (in *Installer) log() *slog.Logger {
+	if in.Log != nil {
+		return in.Log
+	}
+	return slog.Default()
+}
+
+func (in *Installer) client() *http.Client {
+	if in.HTTP != nil {
+		return in.HTTP
+	}
+	return &http.Client{Timeout: 30 * time.Minute}
+}
+
+func (in *Installer) apiBase() string {
+	if in.APIBase != "" {
+		return in.APIBase
+	}
+	return "https://api.github.com"
+}
+
+func (in *Installer) downloadBase() string {
+	if in.DownloadBase != "" {
+		return in.DownloadBase
+	}
+	return "https://github.com"
+}
+
+func lastLine(b []byte) string {
+	lines := strings.Split(strings.TrimSpace(string(b)), "\n")
+	return strings.TrimSpace(lines[len(lines)-1])
+}
+
+func firstVersionLine(b []byte) string {
+	for _, l := range strings.Split(string(b), "\n") {
+		if strings.Contains(strings.ToLower(l), "version") {
+			return strings.TrimSpace(l)
+		}
+	}
+	return ""
+}

internal/agent/provision/install_test.go

@@ -0,0 +1,332 @@
+package provision
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+)
+
+var ubuntu = discovery.Host{OSID: "ubuntu", OSVersion: "24.04", Arch: "amd64", CPUFlags: []string{"avx2"}}
+
+// release is a fake GitHub: one release, its assets and their digests.
+type release struct {
+	assets  map[string][]byte
+	digests map[string]string
+	gets    int
+	mu      sync.Mutex
+}
+
+func tarGz(t *testing.T, files map[string]string, links map[string]string) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	gz := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(gz)
+	for name, body := range files {
+		_ = tw.WriteHeader(&tar.Header{Name: name, Mode: 0o755, Size: int64(len(body)), Typeflag: tar.TypeReg})
+		_, _ = tw.Write([]byte(body))
+	}
+	for name, target := range links {
+		_ = tw.WriteHeader(&tar.Header{Name: name, Linkname: target, Typeflag: tar.TypeSymlink})
+	}
+	_ = tw.Close()
+	_ = gz.Close()
+	return buf.Bytes()
+}
+
+func (r *release) serve(t *testing.T) *httptest.Server {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		r.mu.Lock()
+		r.gets++
+		r.mu.Unlock()
+		if strings.HasPrefix(req.URL.Path, "/repos/") {
+			var out struct {
+				Assets []map[string]string `json:"assets"`
+			}
+			for name := range r.assets {
+				out.Assets = append(out.Assets, map[string]string{"name": name, "digest": r.digests[name]})
+			}
+			_ = json.NewEncoder(w).Encode(out)
+			return
+		}
+		body, ok := r.assets[filepath.Base(req.URL.Path)]
+		if !ok {
+			http.NotFound(w, req)
+			return
+		}
+		_, _ = w.Write(body)
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func newRelease(assets map[string][]byte) *release {
+	r := &release{assets: assets, digests: map[string]string{}}
+	for name, b := range assets {
+		sum := sha256.Sum256(b)
+		r.digests[name] = "sha256:" + hex.EncodeToString(sum[:])
+	}
+	return r
+}
+
+// fakeSystem answers the package tools, ldd and the engine itself.
+type fakeSystem struct {
+	mu        sync.Mutex
+	installed map[string]bool
+	tools     map[string]bool
+	lddOut    string
+	calls     []string
+	// locked is how many more package commands find the lock held.
+	locked int
+}
+
+var errExit = errors.New("exit 1")
+
+func (f *fakeSystem) run(_ context.Context, _ []string, name string, args ...string) ([]byte, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.calls = append(f.calls, name+" "+strings.Join(args, " "))
+	switch {
+	case name == "sh" && strings.Contains(args[1], "dpkg-query"),
+		name == "rpm" && args[0] == "-q":
+		if f.installed[args[len(args)-1]] {
+			return nil, nil
+		}
+		return nil, errExit
+	case name == "sh" && strings.Contains(args[1], "command -v"):
+		if f.tools[args[2]] {
+			return nil, nil
+		}
+		return nil, errExit
+	case name == "apt-get" || name == "dnf":
+		if f.locked > 0 {
+			f.locked--
+			return []byte("E: Unable to lock directory /var/lib/apt/lists/\n"), errExit
+		}
+		for i, a := range args {
+			if a == "install" {
+				for _, p := range args[i+1:] {
+					if !strings.HasPrefix(p, "-") {
+						f.installed[p] = true
+					}
+				}
+			}
+		}
+		return nil, nil
+	case name == "ldd":
+		return []byte(f.lddOut), nil
+	case strings.HasSuffix(name, "llama-server"):
+		return []byte("version: 10991 (abc)\n"), nil
+	}
+	return nil, errors.New("unexpected command " + name)
+}
+
+func installer(t *testing.T, srv *httptest.Server, sys *fakeSystem, host discovery.Host) (*Installer, *[]Status) {
+	var seen []Status
+	var mu sync.Mutex
+	return &Installer{
+		Root: t.TempDir(), Host: host, Run: sys.run,
+		APIBase: srv.URL, DownloadBase: srv.URL, HTTP: srv.Client(),
+		OnStatus: func(s Status) { mu.Lock(); seen = append(seen, s); mu.Unlock() },
+	}, &seen
+}
+
+func llamaRelease(t *testing.T) *release {
+	return newRelease(map[string][]byte{
+		"llama-b1-bin-ubuntu-x64.tar.gz": tarGz(t,
+			map[string]string{"llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libllama.so.0": "x"},
+			map[string]string{"llama-b1/libllama.so": "libllama.so.0"}),
+	})
+}
+
+func TestAnEngineIsInstalledFromNothing(t *testing.T) {
+	// A node rebuilt from a bare image: no engine, no OpenMP runtime. What it
+	// ends with has to have come from the publisher, checked, and been run.
+	rel := llamaRelease(t)
+	sys := &fakeSystem{installed: map[string]bool{}}
+	in, seen := installer(t, rel.serve(t), sys, ubuntu)
+
+	got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
+	if len(got) != 1 || got[0].State != Ready {
+		t.Fatalf("not ready: %+v", got)
+	}
+	if !sys.installed["libgomp1"] {
+		t.Error("the OpenMP runtime the build links against was not installed")
+	}
+	if !strings.HasPrefix(got[0].Binary, in.Root) {
+		t.Errorf("binary %s is not the one installed under %s", got[0].Binary, in.Root)
+	}
+	if _, err := os.Lstat(filepath.Join(filepath.Dir(got[0].Binary), "libllama.so")); err != nil {
+		t.Errorf("the library symlink was not unpacked: %v", err)
+	}
+	var states []string
+	for _, s := range *seen {
+		states = append(states, s.State+": "+s.Detail)
+	}
+	if !strings.Contains(strings.Join(states, "\n"), "downloading") {
+		t.Errorf("a download was not reported while it ran:\n%s", strings.Join(states, "\n"))
+	}
+}
+
+func TestAnInstalledEngineIsCheckedNotFetchedAgain(t *testing.T) {
+	rel := llamaRelease(t)
+	sys := &fakeSystem{installed: map[string]bool{}}
+	in, _ := installer(t, rel.serve(t), sys, ubuntu)
+	w := []Want{{Engine: "llamacpp", Version: "b1"}}
+	in.Converge(t.Context(), w)
+	before := rel.gets
+
+	got := in.Converge(t.Context(), w)
+	if got[0].State != Ready {
+		t.Fatalf("second pass: %+v", got[0])
+	}
+	if rel.gets != before {
+		t.Errorf("a ready engine was downloaded again (%d requests)", rel.gets-before)
+	}
+}
+
+func TestAnEngineThatCannotLoadALibraryIsNotReady(t *testing.T) {
+	// Found on a node before: llama-server present, executable, and unable to
+	// start for want of a library — while the node reported ready.
+	rel := llamaRelease(t)
+	sys := &fakeSystem{installed: map[string]bool{"libgomp1": true},
+		lddOut: "\tlibcuda.so.1 => not found\n"}
+	in, _ := installer(t, rel.serve(t), sys, ubuntu)
+
+	got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
+	if got[0].State != Failed || !strings.Contains(got[0].Detail, "libcuda.so.1") {
+		t.Fatalf("expected a failure naming the library: %+v", got[0])
+	}
+	if _, err := os.Stat(filepath.Join(in.Root, "llamacpp", "b1-cpu", marker)); err == nil {
+		t.Error("a build that does not run was marked installed")
+	}
+}
+
+func TestADownloadThatDoesNotMatchItsChecksumIsRefused(t *testing.T) {
+	rel := llamaRelease(t)
+	for name := range rel.digests {
+		rel.digests[name] = "sha256:" + strings.Repeat("0", 64)
+	}
+	sys := &fakeSystem{installed: map[string]bool{"libgomp1": true}}
+	in, _ := installer(t, rel.serve(t), sys, ubuntu)
+
+	got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
+	if got[0].State != Failed || !strings.Contains(got[0].Detail, "checksum") {
+		t.Fatalf("a tampered download was not refused: %+v", got[0])
+	}
+}
+
+func TestAnArchiveCannotWriteOutsideItsDirectory(t *testing.T) {
+	rel := newRelease(map[string][]byte{
+		"llama-b1-bin-ubuntu-x64.tar.gz": tarGz(t,
+			map[string]string{"../../escaped": "x", "llama-server": ""}, nil),
+	})
+	sys := &fakeSystem{installed: map[string]bool{"libgomp1": true}}
+	in, _ := installer(t, rel.serve(t), sys, ubuntu)
+
+	got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
+	if got[0].State != Failed {
+		t.Fatalf("an archive escaping its directory was installed: %+v", got[0])
+	}
+}
+
+func TestTheBuildFollowsTheHardware(t *testing.T) {
+	gpu := ubuntu
+	gpu.NVIDIADriver = "570.86.15"
+	b, err := chooseLlamaCpp("b1", gpu)
+	if err != nil || !strings.HasPrefix(b.Variant, "cuda") || len(b.Assets) != 2 {
+		t.Errorf("an NVIDIA node got %+v, %v; expected the CUDA build and its runtime", b, err)
+	}
+	if _, err := chooseLlamaCpp("b1", discovery.Host{OSID: "ubuntu", Arch: "riscv64"}); err == nil {
+		t.Error("a build was chosen for an architecture llama.cpp does not publish for")
+	}
+
+	slow := ubuntu
+	slow.CPUFlags = nil
+	st := (&Installer{Host: slow}).ready(Status{}, Build{CPUFlags: []string{"avx2"}}, "/x/llama-server", "")
+	if !strings.Contains(st.Detail, "avx2") {
+		t.Errorf("a CPU without AVX2 was not called out: %q", st.Detail)
+	}
+}
+
+func TestAnEngineTheAgentCannotInstallSaysSo(t *testing.T) {
+	in := &Installer{Root: t.TempDir(), Host: ubuntu}
+	got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.29.0"}})
+	if got[0].State != Unsupported {
+		t.Errorf("got %+v", got[0])
+	}
+}
+
+func TestARedHatFamilyNodeInstallsWithItsOwnPackageManager(t *testing.T) {
+	// The build is not Ubuntu's in any way that matters; the package that
+	// carries OpenMP's runtime is named differently, and installed by dnf.
+	rocky := discovery.Host{OSID: "rocky", OSVersion: "10.0", OSLike: []string{"rhel", "centos", "fedora"},
+		Arch: "amd64", CPUFlags: []string{"avx2"}}
+	rel := llamaRelease(t)
+	sys := &fakeSystem{installed: map[string]bool{}, tools: map[string]bool{"dnf": true}}
+	in, _ := installer(t, rel.serve(t), sys, rocky)
+
+	got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
+	if got[0].State != Ready {
+		t.Fatalf("not ready: %+v", got[0])
+	}
+	if !sys.installed["libgomp"] || sys.installed["libgomp1"] {
+		t.Errorf("installed %v; expected the rpm name, libgomp", sys.installed)
+	}
+}
+
+func TestASystemTooOldForTheBuildSaysWhatItLacks(t *testing.T) {
+	// RHEL 9: glibc is new enough, libstdc++ is one GCC release short. ldd
+	// finds the library and reports the symbol, and "libstdc++ not found"
+	// would send somebody looking for a package that is installed.
+	rel := llamaRelease(t)
+	sys := &fakeSystem{installed: map[string]bool{"libgomp1": true},
+		lddOut: "\t/lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by /x/libllama.so)\n" +
+			"\t/lib64/libstdc++.so.6: version `GLIBCXX_3.4.30' not found (required by /x/llama-server)\n"}
+	in, _ := installer(t, rel.serve(t), sys, ubuntu)
+
+	got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
+	if got[0].State != Failed || !strings.Contains(got[0].Detail, "libstdc++.so.6 is too old (needs GLIBCXX_3.4.30)") {
+		t.Fatalf("detail %q", got[0].Detail)
+	}
+	if strings.Count(got[0].Detail, "GLIBCXX") != 1 {
+		t.Errorf("the same shortfall was listed more than once: %q", got[0].Detail)
+	}
+}
+
+func TestAPackageManagerBusyWithItsOwnUpdatesIsWaitedFor(t *testing.T) {
+	// A new node's first minutes: cloud-init and unattended-upgrades hold
+	// apt's lists lock, which DPkg::Lock::Timeout does not cover.
+	lockRetry = time.Millisecond
+	t.Cleanup(func() { lockRetry = 10 * time.Second })
+
+	rel := llamaRelease(t)
+	sys := &fakeSystem{installed: map[string]bool{}, locked: 3}
+	in, seen := installer(t, rel.serve(t), sys, ubuntu)
+
+	got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
+	if got[0].State != Ready {
+		t.Fatalf("gave up on a lock another apt held: %+v", got[0])
+	}
+	waited := false
+	for _, s := range *seen {
+		waited = waited || strings.Contains(s.Detail, "waiting for another apt-get")
+	}
+	if !waited {
+		t.Error("the wait was not reported")
+	}
+}

internal/agent/provision/recipe.go

@@ -0,0 +1,109 @@
+// Package provision installs the engines a node is asked to have.
+//
+// The control plane says which engines at which versions; everything else is
+// worked out here, on the node, because only the node knows it: which build
+// fits its processor and driver, what its operating system is missing, and
+// whether the result actually runs. Nothing already on the machine is taken
+// on trust. An engine counts as installed when this package installed it,
+// checked every file against the checksum its publisher gave, and watched it
+// start.
+package provision
+
+import (
+	"fmt"
+	"strings"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+)
+
+// Build is one published build of an engine, chosen for one machine.
+type Build struct {
+	// Variant names the build: "cpu", "cuda-12.8".
+	Variant string
+	// Assets are the release files to fetch, unpacked in order into one
+	// directory.
+	Assets []string
+	// Packages are operating system packages the build links against and a
+	// minimal install may not carry, by packaging family ("deb", "rpm"): the
+	// same library is libgomp1 on one and libgomp on the other.
+	Packages map[string][]string
+	// Binary is the executable's file name, found anywhere in the unpacked
+	// tree.
+	Binary string
+	// CPUFlags are extensions the build is slow without. Missing ones are
+	// reported, not refused: slow is still running.
+	CPUFlags []string
+}
+
+// Recipe knows how to install one engine.
+type Recipe struct {
+	// Repo is the GitHub repository whose releases carry the builds.
+	Repo string
+	// Choose picks the build for a machine, or says why none fits.
+	Choose func(version string, h discovery.Host) (Build, error)
+}
+
+// Recipes are the engines this agent can install.
+//
+// vLLM and SGLang are not here yet: they are Python environments of several
+// gigabytes rather than an archive, and are installed by hand and named in the
+// agent's configuration until they are.
+var Recipes = map[string]Recipe{
+	"llamacpp": {Repo: "ggml-org/llama.cpp", Choose: chooseLlamaCpp},
+}
+
+// cudaForLlamaCpp is the CUDA build taken on an NVIDIA node. The release also
+// carries a matching cudart archive, so the node needs a driver and nothing
+// else — no toolkit.
+const cudaForLlamaCpp = "12.8"
+
+func chooseLlamaCpp(version string, h discovery.Host) (Build, error) {
+	// The published Linux builds are built on Ubuntu, but nothing about them
+	// is Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which
+	// Debian 12 and RHEL 10 have and RHEL 9 does not. So the distribution's
+	// name decides nothing here. The build is tried everywhere, and ldd says
+	// whether this system can load it — in the words of the missing symbol.
+	var arch string
+	switch h.Arch {
+	case "amd64":
+		arch = "x64"
+	case "arm64":
+		arch = "arm64"
+	default:
+		return Build{}, fmt.Errorf("llama.cpp publishes no Linux build for %s", orUnknown(h.Arch))
+	}
+
+	b := Build{
+		Variant: "cpu",
+		Assets:  []string{fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)},
+		// Everything else the build links is in the archive or in any current
+		// glibc system; OpenMP's runtime is the one a cloud image lacks, and
+		// the server does not start without it.
+		Packages: map[string][]string{"deb": {"libgomp1"}, "rpm": {"libgomp"}},
+		Binary:   "llama-server",
+	}
+	if arch == "x64" {
+		// The archive carries a backend per CPU generation and loads the best
+		// one, down to SSE4.2. Below AVX2 it runs, several times slower.
+		b.CPUFlags = []string{"avx2"}
+	}
+	// The CUDA build is published for x64 at this version; on arm64 it is a
+	// newer CUDA that the driver may not support, so an arm64 GPU node runs
+	// the CPU build until someone decides otherwise.
+	if h.NVIDIADriver != "" && arch == "x64" {
+		name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp)
+		b.Variant = "cuda-" + cudaForLlamaCpp
+		b.Assets = []string{
+			fmt.Sprintf("llama-%s-%s", version, name),
+			fmt.Sprintf("cudart-llama-%s-%s", version, name),
+		}
+	}
+	return b, nil
+}
+
+func orUnknown(s string) string {
+	if strings.TrimSpace(s) == "" {
+		return "an unrecognised system"
+	}
+	return s
+}

internal/agent/removeweights_test.go

@@ -0,0 +1,96 @@
+package agent
+
+import (
+	"log/slog"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
+)
+
+// agentWithCache is an agent that only knows where its weights live, which is
+// all removeWeights needs.
+func agentWithCache(t *testing.T, dir string) *Agent {
+	t.Helper()
+	return &Agent{
+		cfg: Config{ModelDirs: []string{dir}},
+		log: slog.New(slog.DiscardHandler),
+	}
+}
+
+func write(t *testing.T, path string) string {
+	t.Helper()
+	if err := os.WriteFile(path, []byte("weights"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return path
+}
+
+func TestTheAgentRefusesToDeleteWhatItDidNotFetch(t *testing.T) {
+	// The control plane computes these paths, but the node holds the files. A
+	// control plane that is wrong, newer than this agent, or tampered with must
+	// not be able to talk it into deleting a model an operator spent a week
+	// copying here. So the agent checks again, on its own.
+	dir := t.TempDir()
+	a := agentWithCache(t, dir)
+
+	operators := write(t, filepath.Join(dir, "qwen2.5-coder-14b-instruct-q8_0.gguf"))
+	fetched := write(t, weights.CachePathFor(dir, "https://models.internal/gone.gguf"))
+
+	a.removeWeights([]string{operators, fetched})
+
+	if _, err := os.Stat(operators); err != nil {
+		t.Errorf("a file llmgw never fetched was deleted: %v", err)
+	}
+	if _, err := os.Stat(fetched); !os.IsNotExist(err) {
+		t.Errorf("the fetched file was not removed: %v", err)
+	}
+}
+
+func TestTheAgentRefusesPathsOutsideItsCache(t *testing.T) {
+	// Even with a name that looks like one of ours. The cache directory is the
+	// boundary, and traversal out of it is the attack this closes.
+	dir := t.TempDir()
+	elsewhere := t.TempDir()
+	a := agentWithCache(t, dir)
+
+	// A correctly named cache file, in the wrong directory.
+	outside := write(t, weights.CachePathFor(elsewhere, "https://models.internal/x.gguf"))
+	// And the same thing reached by traversal from inside the cache.
+	traversal := filepath.Join(dir, "..", filepath.Base(elsewhere),
+		filepath.Base(outside))
+
+	a.removeWeights([]string{outside, traversal})
+
+	if _, err := os.Stat(outside); err != nil {
+		t.Errorf("a file outside the cache was deleted: %v", err)
+	}
+}
+
+func TestRemovingAFileThatIsAlreadyGoneIsNotAFailure(t *testing.T) {
+	// The expected outcome of a retry. It must not stop the rest of the batch.
+	dir := t.TempDir()
+	a := agentWithCache(t, dir)
+
+	missing := weights.CachePathFor(dir, "https://models.internal/never.gguf")
+	present := write(t, weights.CachePathFor(dir, "https://models.internal/here.gguf"))
+
+	a.removeWeights([]string{missing, present})
+
+	if _, err := os.Stat(present); !os.IsNotExist(err) {
+		t.Errorf("a missing file stopped the batch: %v", err)
+	}
+}
+
+func TestWithNoCacheConfiguredNothingIsDeleted(t *testing.T) {
+	dir := t.TempDir()
+	a := &Agent{cfg: Config{}, log: slog.New(slog.DiscardHandler)}
+	kept := write(t, weights.CachePathFor(dir, "https://models.internal/x.gguf"))
+
+	a.removeWeights([]string{kept})
+
+	if _, err := os.Stat(kept); err != nil {
+		t.Errorf("an agent with no cache directory deleted something: %v", err)
+	}
+}

internal/agent/selfdescribe_test.go

@@ -0,0 +1,119 @@
+package agent
+
+import (
+	"context"
+	"os"
+	"slices"
+	"testing"
+	"time"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+)
+
+func TestAnAgentConfigNeedsOnlyWhereTheControlPlaneIs(t *testing.T) {
+	// Every node carried its own name, its own address and the same two
+	// directories written out by hand, so no two agent files were alike and a
+	// copy taken from one node registered the next as the first.
+	cfg := Config{Server: "127.0.0.1:7443"}
+	if err := cfg.normalize(); err != nil {
+		t.Fatal(err)
+	}
+	host, _ := os.Hostname()
+	if cfg.NodeID != host || cfg.Hostname != host {
+		t.Errorf("node_id %q, hostname %q; expected the machine's name %q",
+			cfg.NodeID, cfg.Hostname, host)
+	}
+	if cfg.MgmtAddr != "127.0.0.1" {
+		t.Errorf("mgmt_addr %q, expected the address on the route to the control plane",
+			cfg.MgmtAddr)
+	}
+	if cfg.WeightsCache != DefaultWeightsCache {
+		t.Errorf("weights land in %q", cfg.WeightsCache)
+	}
+	if !slices.Contains(cfg.ModelDirs, cfg.WeightsCache) {
+		t.Errorf("the download directory %q is not looked in: %v",
+			cfg.WeightsCache, cfg.ModelDirs)
+	}
+}
+
+func TestWhatIsWrittenStillWins(t *testing.T) {
+	cfg := Config{
+		Server: "127.0.0.1:7443", NodeID: "gpu-a-01", MgmtAddr: "10.9.9.9",
+		ModelDirs: []string{"/lustre/models"}, WeightsCache: "/scratch/w",
+	}
+	if err := cfg.normalize(); err != nil {
+		t.Fatal(err)
+	}
+	if cfg.NodeID != "gpu-a-01" || cfg.MgmtAddr != "10.9.9.9" {
+		t.Errorf("configured identity overridden: %q %q", cfg.NodeID, cfg.MgmtAddr)
+	}
+	if !slices.Equal(cfg.ModelDirs, []string{"/lustre/models", "/scratch/w"}) {
+		t.Errorf("model dirs %v", cfg.ModelDirs)
+	}
+}
+
+func TestAHeartbeatSaysWhenTheEnginesChanged(t *testing.T) {
+	cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir()}
+	cfg.Superv.WorkDir = t.TempDir()
+	a, err := New(cfg, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	engines := func(m *llmgwv1.AgentMessage) *llmgwv1.Heartbeat { return m.GetHeartbeat() }
+
+	a.mu.Lock()
+	a.configuredVers = map[string]string{"llamacpp": "unknown"}
+	a.enginesSent = enginesFingerprint(a.configuredVers)
+	a.mu.Unlock()
+	if hb := engines(a.heartbeat()); hb.GetEnginesChanged() {
+		t.Errorf("unchanged engines were resent")
+	}
+
+	a.mu.Lock()
+	a.configuredVers = map[string]string{"llamacpp": "unknown", "vllm": "unknown"}
+	a.mu.Unlock()
+	hb := engines(a.heartbeat())
+	if !hb.GetEnginesChanged() || len(hb.GetEngineVersions()) != 2 {
+		t.Errorf("an installed engine was not reported: %v", hb.GetEngineVersions())
+	}
+	if engines(a.heartbeat()).GetEnginesChanged() {
+		t.Errorf("reported again on the next heartbeat")
+	}
+}
+
+func TestWhatTheControlPlaneAsksToInstallIsReportedBack(t *testing.T) {
+	cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(),
+		EnginesDir: t.TempDir()}
+	cfg.Superv.WorkDir = t.TempDir()
+	a, err := New(cfg, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	go a.provisionLoop(ctx)
+
+	// An engine this agent cannot install: no network is needed to learn so,
+	// and the answer has to reach the control plane rather than a log.
+	a.setProvision(&llmgwv1.Provision{Engines: []*llmgwv1.EngineWant{{Engine: "vllm", Version: "0.29.0"}}})
+
+	deadline := time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		hb := a.heartbeat().GetHeartbeat()
+		if hb.GetProvisionChanged() {
+			p := hb.GetProvision()
+			if len(p) != 1 || p[0].GetState() != "unsupported" {
+				t.Fatalf("reported %v", p)
+			}
+			a.mu.Lock()
+			_, reported := a.engineReportLocked()["vllm"]
+			a.mu.Unlock()
+			if reported {
+				t.Error("an engine that was not installed is reported as present")
+			}
+			return
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	t.Fatal("the installation's outcome never reached a heartbeat")
+}

internal/agent/servers_test.go

@@ -0,0 +1,90 @@
+package agent
+
+import (
+	"slices"
+	"strings"
+	"testing"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
+)
+
+func normalized(t *testing.T, c Config) Config {
+	t.Helper()
+	if err := c.normalizeServerURL(); err != nil {
+		t.Fatalf("normalising %+v: %v", c, err)
+	}
+	return c
+}
+
+func TestOneServerStillWorksExactlyAsBefore(t *testing.T) {
+	// Every agent configuration written before there could be a second control
+	// plane names one address, and none of them should have to change.
+	c := normalized(t, Config{Server: "10.0.0.140:7443"})
+	if !slices.Equal(c.Servers, []string{"http://10.0.0.140:7443"}) {
+		t.Errorf("servers = %v", c.Servers)
+	}
+	if c.Server != "http://10.0.0.140:7443" {
+		t.Errorf("server = %q", c.Server)
+	}
+}
+
+func TestTheAddressAlreadyThereStaysPreferred(t *testing.T) {
+	// An operator adding a standby to an existing file expects the control
+	// plane that was already working to keep being tried first.
+	c := normalized(t, Config{
+		Server:  "first:7443",
+		Servers: []string{"second:7443", "third:7443"},
+	})
+	want := []string{"http://first:7443", "http://second:7443", "http://third:7443"}
+	if !slices.Equal(c.Servers, want) {
+		t.Errorf("servers = %v, want %v", c.Servers, want)
+	}
+}
+
+func TestADuplicateAddressIsNotTriedTwiceInARow(t *testing.T) {
+	// Which would look exactly like the rotation not working.
+	c := normalized(t, Config{
+		Server:  "a:7443",
+		Servers: []string{"http://a:7443", "b:7443"},
+	})
+	want := []string{"http://a:7443", "http://b:7443"}
+	if !slices.Equal(c.Servers, want) {
+		t.Errorf("servers = %v, want %v", c.Servers, want)
+	}
+}
+
+func TestEveryAddressIsCheckedAgainstTheTLSSetting(t *testing.T) {
+	// The check existed for one address and has to hold for all of them: a
+	// standby reached over plaintext while the leader is authenticated would
+	// be a way around mutual TLS, not a fallback.
+	c := Config{
+		Servers: []string{"https://a:7443", "http://b:7443"},
+		TLS:     tlsconf.Config{CA: "ca.pem", Cert: "c.pem", Key: "k.pem"},
+	}
+	err := c.normalizeServerURL()
+	if err == nil {
+		t.Fatal("a plaintext standby was accepted alongside TLS")
+	}
+	if !strings.Contains(err.Error(), "b:7443") {
+		t.Errorf("the error does not say which address: %v", err)
+	}
+}
+
+func TestNoServerAtAllIsStillAnError(t *testing.T) {
+	c := Config{}
+	if err := c.normalizeServerURL(); err == nil {
+		t.Error("an agent with no control plane configured was accepted")
+	}
+}
+
+func TestSchemeIsAddedPerAddress(t *testing.T) {
+	c := normalized(t, Config{
+		Servers: []string{"a:7443", "b:7443"},
+		TLS:     tlsconf.Config{CA: "ca.pem", Cert: "c.pem", Key: "k.pem"},
+	})
+	for _, s := range c.Servers {
+		if !strings.HasPrefix(s, "https://") {
+			t.Errorf("%q did not get the secure scheme", s)
+		}
+	}
+}

internal/agent/shutdown_test.go

@@ -0,0 +1,59 @@
+package agent
+
+import (
+	"context"
+	"log/slog"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
+)
+
+// TestAgentStopsWhenContextEnds pins down something cancellation does not do by
+// itself. A bidirectional stream spends its life blocked reading, and HTTP/2
+// stops watching the context once the request headers are away, so an agent
+// that only cancels keeps reading until something kills it. Under systemd that
+// is the full stop timeout on every restart, followed by SIGKILL.
+func TestAgentStopsWhenContextEnds(t *testing.T) {
+	serverTLS, clientTLS := pki(t, "llmgw-node01")
+	svc, inv, url := startTLS(t, serverTLS)
+
+	cfg := DefaultConfig()
+	cfg.NodeID = "llmgw-node01"
+	cfg.Server = url
+	cfg.TLS = clientTLS
+	cfg.HeartbeatInterval = 50 * time.Millisecond
+	cfg.Discovery.SkipGPUs = true
+	cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent")
+	cfg.Superv = supervisor.Config{
+		WorkDir: t.TempDir(), LogLines: 8,
+		StopTimeout: time.Second, HealthInterval: 20 * time.Millisecond,
+	}
+
+	client, err := Client(cfg.TLS, 5*time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ag, err := New(cfg, client, slog.New(slog.DiscardHandler))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { _ = ag.Run(ctx); close(done) }()
+
+	// Cancelling before the stream is established would prove nothing: the
+	// blocked read is the case that matters.
+	if !eventually(t, 10*time.Second, registered(svc, inv, "llmgw-node01")) {
+		t.Fatal("the agent never registered")
+	}
+
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the agent did not stop within 5s of being told to")
+	}
+}

internal/agent/supervisor/health.go

@@ -0,0 +1,99 @@
+package supervisor
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+// HTTPHealth checks an engine over HTTP.
+//
+// Readiness is two questions, not one (plan §5.4): /health says the process is
+// up, and /v1/models says the weights finished loading and the model is being
+// served. An engine answers /health long before it can serve a request, so
+// routing on /health alone sends traffic into a model that is still loading.
+type HTTPHealth struct {
+	client      *http.Client
+	healthPath  string
+	modelsPath  string
+	requireName bool
+}
+
+// HTTPHealthOption customizes the checker.
+type HTTPHealthOption func(*HTTPHealth)
+
+// WithHealthPath overrides the readiness path.
+func WithHealthPath(p string) HTTPHealthOption {
+	return func(h *HTTPHealth) { h.healthPath = p }
+}
+
+// WithoutModelCheck skips the /v1/models step, for engines that do not expose it.
+func WithoutModelCheck() HTTPHealthOption {
+	return func(h *HTTPHealth) { h.requireName = false }
+}
+
+// NewHTTPHealth builds a checker.
+func NewHTTPHealth(timeout time.Duration, opts ...HTTPHealthOption) *HTTPHealth {
+	if timeout <= 0 {
+		timeout = 3 * time.Second
+	}
+	h := &HTTPHealth{
+		client:      &http.Client{Timeout: timeout},
+		healthPath:  "/health",
+		modelsPath:  "/v1/models",
+		requireName: true,
+	}
+	for _, o := range opts {
+		o(h)
+	}
+	return h
+}
+
+// Ready reports whether the engine at baseURL is serving model.
+func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
+	if err := h.get(ctx, baseURL+h.healthPath, nil); err != nil {
+		return fmt.Errorf("health: %w", err)
+	}
+	if !h.requireName || model == "" {
+		return nil
+	}
+
+	var list struct {
+		Data []struct {
+			ID string `json:"id"`
+		} `json:"data"`
+	}
+	if err := h.get(ctx, baseURL+h.modelsPath, &list); err != nil {
+		return fmt.Errorf("models: %w", err)
+	}
+	for _, d := range list.Data {
+		if d.ID == model {
+			return nil
+		}
+	}
+	return fmt.Errorf("engine is up but does not serve %q yet", model)
+}
+
+func (h *HTTPHealth) get(ctx context.Context, url string, out any) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := h.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<10))
+		return fmt.Errorf("status %s", resp.Status)
+	}
+	if out == nil {
+		return nil
+	}
+	return json.NewDecoder(resp.Body).Decode(out)
+}

internal/agent/supervisor/health_test.go

@@ -0,0 +1,86 @@
+package supervisor
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+// engineStub serves configurable /health and /v1/models responses.
+func engineStub(t *testing.T, healthy bool, models ...string) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
+		if !healthy {
+			http.Error(w, "loading", http.StatusServiceUnavailable)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	})
+	mux.HandleFunc("GET /v1/models", func(w http.ResponseWriter, _ *http.Request) {
+		var b strings.Builder
+		b.WriteString(`{"object":"list","data":[`)
+		for i, m := range models {
+			if i > 0 {
+				b.WriteString(",")
+			}
+			b.WriteString(`{"id":"` + m + `"}`)
+		}
+		b.WriteString(`]}`)
+		io.WriteString(w, b.String())
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestReadyWhenServingModel(t *testing.T) {
+	srv := engineStub(t, true, "llama-3.3-70b")
+	if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b"); err != nil {
+		t.Errorf("a serving engine was reported unready: %v", err)
+	}
+}
+
+func TestNotReadyWhileLoading(t *testing.T) {
+	// The case that matters: the process is up but the weights are not loaded.
+	// Routing on /health alone would send traffic into a loading model.
+	srv := engineStub(t, false, "llama-3.3-70b")
+	err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b")
+	if err == nil {
+		t.Fatal("an unhealthy engine was reported ready")
+	}
+	if !strings.Contains(err.Error(), "health") {
+		t.Errorf("err=%v", err)
+	}
+}
+
+func TestNotReadyWhenModelMissing(t *testing.T) {
+	// /health passes but the model is not listed: the engine is up and serving
+	// something else, or still registering.
+	srv := engineStub(t, true, "some-other-model")
+	err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b")
+	if err == nil {
+		t.Fatal("a model that is not served was reported ready")
+	}
+	if !strings.Contains(err.Error(), "does not serve") {
+		t.Errorf("err=%v", err)
+	}
+}
+
+func TestModelCheckCanBeSkipped(t *testing.T) {
+	srv := engineStub(t, true)
+	h := NewHTTPHealth(time.Second, WithoutModelCheck())
+	if err := h.Ready(t.Context(), srv.URL, "anything"); err != nil {
+		t.Errorf("the model check was not skipped: %v", err)
+	}
+}
+
+func TestUnreachableEngine(t *testing.T) {
+	h := NewHTTPHealth(200 * time.Millisecond)
+	if err := h.Ready(t.Context(), "http://127.0.0.1:1", "m"); err == nil {
+		t.Error("an unreachable engine was reported ready")
+	}
+}

internal/agent/supervisor/process_unix.go

@@ -0,0 +1,65 @@
+package supervisor
+
+import (
+	"bytes"
+	"os"
+	"strconv"
+	"syscall"
+)
+
+// killGroup signals a whole process group.
+//
+// Engines fork worker processes that hold GPU memory. Signalling only the
+// leader leaves those workers alive, and the next launch then fails to
+// allocate — with an error that points at the new process rather than the old.
+func killGroup(pid int, sig syscall.Signal) error {
+	// The negative pid addresses the group. Setpgid at launch made the leader's
+	// pid the group id.
+	if err := syscall.Kill(-pid, sig); err != nil {
+		if err == syscall.ESRCH {
+			return os.ErrProcessDone
+		}
+		return err
+	}
+	return nil
+}
+
+// processAlive reports whether a pid belongs to a process that is still doing
+// something.
+//
+// A zombie is not. It has exited and is only waiting to be reaped, but
+// kill(pid, 0) still succeeds on it — so a naive check reports a dead engine as
+// running forever. That happens for real whenever an engine's parent dies
+// before it does and nothing reaps the orphan: in a container, PID 1 is usually
+// the workload rather than an init, and orphans simply accumulate.
+func processAlive(pid int) bool {
+	if pid <= 0 {
+		return false
+	}
+	// Signal 0 performs the permission and existence checks without delivering
+	// anything.
+	if err := syscall.Kill(pid, 0); err != nil {
+		// EPERM means it exists but belongs to someone else, which for our
+		// purposes is still "there".
+		return err == syscall.EPERM
+	}
+	return !isZombie(pid)
+}
+
+// isZombie reports whether a pid has exited but not been reaped.
+//
+// Only Linux exposes this; elsewhere there is no procfs and the answer is
+// false, which degrades to the old behaviour rather than to something worse.
+func isZombie(pid int) bool {
+	b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat")
+	if err != nil {
+		return false
+	}
+	// The comm field is parenthesised and may itself contain spaces or
+	// parentheses, so the state character is found from the last ')'.
+	i := bytes.LastIndexByte(b, ')')
+	if i < 0 || i+2 >= len(b) {
+		return false
+	}
+	return b[i+2] == 'Z'
+}

internal/agent/supervisor/ringbuffer.go

@@ -0,0 +1,57 @@
+package supervisor
+
+import "sync"
+
+// ringBuffer keeps the last n lines of output.
+//
+// Engine logs are the only diagnosis available when a shard fails to load, but
+// they are also unbounded — a model loading for ten minutes can produce
+// megabytes. Keeping a bounded tail makes `ctl logs` useful without letting a
+// chatty engine consume the agent's memory.
+type ringBuffer struct {
+	mu    sync.Mutex
+	lines []string
+	next  int
+	full  bool
+}
+
+func newRingBuffer(n int) *ringBuffer {
+	if n <= 0 {
+		n = 256
+	}
+	return &ringBuffer{lines: make([]string, n)}
+}
+
+func (r *ringBuffer) add(line string) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.lines[r.next] = line
+	r.next = (r.next + 1) % len(r.lines)
+	if r.next == 0 {
+		r.full = true
+	}
+}
+
+// tail returns up to n most recent lines, oldest first.
+func (r *ringBuffer) tail(n int) []string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	size := r.next
+	if r.full {
+		size = len(r.lines)
+	}
+	if n <= 0 || n > size {
+		n = size
+	}
+
+	out := make([]string, 0, n)
+	for i := size - n; i < size; i++ {
+		idx := i
+		if r.full {
+			idx = (r.next + i) % len(r.lines)
+		}
+		out = append(out, r.lines[idx])
+	}
+	return out
+}

internal/agent/supervisor/state.go

@@ -0,0 +1,57 @@
+// Package supervisor runs and watches engine processes (plan §5.4).
+package supervisor
+
+import "fmt"
+
+// State is where a shard is in its lifecycle.
+//
+// Loading is separate from Starting because model loading dominates startup:
+// a 70B model takes minutes, and an operator watching a shard sit in one state
+// for ten minutes needs to know whether that is normal.
+type State int
+
+const (
+	// Pending means the shard is known but not started.
+	Pending State = iota
+	// Starting means the process has been launched but has not opened its port.
+	Starting
+	// Loading means the port is open but the model is not ready.
+	Loading
+	// Ready means health checks pass and the model is served.
+	Ready
+	// Draining means no new work; existing requests finish.
+	Draining
+	// Stopped means the process exited as intended.
+	Stopped
+	// Failed means the process exited unexpectedly or never became ready.
+	Failed
+)
+
+func (s State) String() string {
+	switch s {
+	case Pending:
+		return "pending"
+	case Starting:
+		return "starting"
+	case Loading:
+		return "loading"
+	case Ready:
+		return "ready"
+	case Draining:
+		return "draining"
+	case Stopped:
+		return "stopped"
+	case Failed:
+		return "failed"
+	default:
+		return fmt.Sprintf("unknown(%d)", int(s))
+	}
+}
+
+// Terminal reports whether the shard has finished for good.
+func (s State) Terminal() bool { return s == Stopped || s == Failed }
+
+// Live reports whether a process should be running in this state.
+func (s State) Live() bool {
+	return s == Starting || s == Loading || s == Ready || s == Draining
+}

internal/agent/supervisor/supervisor.go

@@ -0,0 +1,771 @@
+package supervisor
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+)
+
+// Config configures the supervisor.
+type Config struct {
+	// WorkDir holds pid files and per-shard scratch space.
+	WorkDir string `koanf:"work_dir"`
+	// LogLines is how many lines of engine output to retain per shard.
+	LogLines int `koanf:"log_lines"`
+	// StopTimeout is how long a process gets to exit after SIGTERM before it is
+	// killed. Engines need time to free GPU memory cleanly.
+	StopTimeout time.Duration `koanf:"stop_timeout"`
+	// HealthInterval is how often a starting shard is polled.
+	HealthInterval time.Duration `koanf:"health_interval"`
+	// RendezvousTimeout bounds the wait for a multi-node rendezvous point to
+	// start listening.
+	RendezvousTimeout time.Duration `koanf:"rendezvous_timeout"`
+	// AliveGrace is how long a non-serving rank must stay up before it counts
+	// as ready. There is no external signal for a worker, so staying alive is
+	// the only evidence available.
+	AliveGrace time.Duration `koanf:"alive_grace"`
+}
+
+// DefaultConfig returns sensible defaults.
+func DefaultConfig() Config {
+	return Config{
+		WorkDir:           "/var/lib/llmgw-agent",
+		LogLines:          512,
+		StopTimeout:       60 * time.Second,
+		HealthInterval:    2 * time.Second,
+		RendezvousTimeout: 5 * time.Minute,
+		AliveGrace:        5 * time.Second,
+	}
+}
+
+func (c *Config) normalize() {
+	d := DefaultConfig()
+	if c.WorkDir == "" {
+		c.WorkDir = d.WorkDir
+	}
+	if c.LogLines <= 0 {
+		c.LogLines = d.LogLines
+	}
+	if c.StopTimeout <= 0 {
+		c.StopTimeout = d.StopTimeout
+	}
+	if c.HealthInterval <= 0 {
+		c.HealthInterval = d.HealthInterval
+	}
+	if c.RendezvousTimeout <= 0 {
+		c.RendezvousTimeout = d.RendezvousTimeout
+	}
+	if c.AliveGrace <= 0 {
+		c.AliveGrace = d.AliveGrace
+	}
+}
+
+// HealthChecker reports whether a shard is serving. It is an interface so the
+// supervisor can be tested without an HTTP engine.
+type HealthChecker interface {
+	// Ready reports whether the shard at baseURL serves model.
+	Ready(ctx context.Context, baseURL, model string) error
+}
+
+// Shard is one supervised engine process.
+type Shard struct {
+	ID   string
+	Spec engines.ShardSpec
+
+	mu       sync.Mutex
+	state    State
+	pid      int
+	err      error
+	started  time.Time
+	readyAt  time.Time
+	exitCode int
+
+	logs *ringBuffer
+	cmd  *exec.Cmd
+	// sidecars are the shard's supporting processes, torn down with it.
+	sidecars []*exec.Cmd
+	cancel   context.CancelFunc
+	done     chan struct{}
+	// adopted marks a shard inherited from a previous agent: it has no
+	// exec.Cmd, so it is watched by polling and killed by signal alone.
+	adopted bool
+}
+
+// Status is a snapshot of a shard, safe to send to the control plane.
+type Status struct {
+	ID       string
+	State    State
+	PID      int
+	Err      string
+	Started  time.Time
+	ReadyAt  time.Time
+	ExitCode int
+}
+
+// Status returns the shard's current status.
+func (s *Shard) Status() Status {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	st := Status{
+		ID: s.ID, State: s.state, PID: s.pid,
+		Started: s.started, ReadyAt: s.readyAt, ExitCode: s.exitCode,
+	}
+	if s.err != nil {
+		st.Err = s.err.Error()
+	}
+	return st
+}
+
+// Logs returns the last n lines of engine output.
+func (s *Shard) Logs(n int) []string { return s.logs.tail(n) }
+
+func (s *Shard) setState(st State, err error) {
+	s.mu.Lock()
+	s.state = st
+	if err != nil {
+		s.err = err
+	}
+	if st == Ready && s.readyAt.IsZero() {
+		s.readyAt = time.Now()
+	}
+	s.mu.Unlock()
+}
+
+// Supervisor starts, watches and stops engine processes on this node.
+type Supervisor struct {
+	cfg    Config
+	log    *slog.Logger
+	health HealthChecker
+
+	mu     sync.Mutex
+	shards map[string]*Shard
+}
+
+// New builds a supervisor.
+func New(cfg Config, health HealthChecker, log *slog.Logger) (*Supervisor, error) {
+	cfg.normalize()
+	if log == nil {
+		log = slog.Default()
+	}
+	if err := os.MkdirAll(cfg.WorkDir, 0o750); err != nil {
+		return nil, fmt.Errorf("work dir: %w", err)
+	}
+	return &Supervisor{cfg: cfg, log: log, health: health, shards: map[string]*Shard{}}, nil
+}
+
+// ErrAlreadyRunning means a shard with that ID is already supervised.
+var ErrAlreadyRunning = errors.New("shard is already running")
+
+// Start launches a shard.
+//
+// It is idempotent by ID: a repeated StartShard for the same shard is a no-op
+// rather than a second process, because the control plane retries commands
+// (plan §5.6).
+func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmds []engines.Command) (*Shard, error) {
+	if len(cmds) == 0 {
+		return nil, fmt.Errorf("shard %s: no commands to run", id)
+	}
+	s.mu.Lock()
+	if existing, ok := s.shards[id]; ok {
+		s.mu.Unlock()
+		if existing.Status().State.Terminal() {
+			s.Remove(id)
+		} else {
+			return existing, ErrAlreadyRunning
+		}
+		s.mu.Lock()
+	}
+
+	sh := &Shard{
+		ID:      id,
+		Spec:    spec,
+		state:   Pending,
+		logs:    newRingBuffer(s.cfg.LogLines),
+		started: time.Now(),
+		done:    make(chan struct{}),
+	}
+	s.shards[id] = sh
+	s.mu.Unlock()
+
+	if err := s.launch(ctx, sh, cmds); err != nil {
+		sh.setState(Failed, err)
+		close(sh.done)
+		return sh, err
+	}
+	return sh, nil
+}
+
+// launch starts a shard's processes in order.
+//
+// Every command but the last is a sidecar: it keeps running for the shard's
+// lifetime and is torn down with it, but its exit is not what defines the
+// shard's. The last command is the engine, and its exit is the shard's exit.
+func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Command) error {
+	runCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))
+	sh.cancel = cancel
+
+	for i, c := range cmds[:len(cmds)-1] {
+		proc, err := s.spawn(runCtx, sh, c)
+		if err != nil {
+			cancel()
+			return fmt.Errorf("sidecar %d (%s): %w", i, c.Name, err)
+		}
+		sh.mu.Lock()
+		sh.sidecars = append(sh.sidecars, proc)
+		sh.mu.Unlock()
+
+		if c.WaitForLeaderPort {
+			// The next process cannot start until the rendezvous point is
+			// listening: a Ray worker cannot join a head that is not up, and
+			// the leader's server cannot claim GPUs from a cluster that does
+			// not exist yet.
+			addr := fmt.Sprintf("%s:%d", sh.Spec.LeaderAddr, sh.Spec.LeaderPort)
+			if err := waitForPort(runCtx, addr, s.cfg.RendezvousTimeout); err != nil {
+				cancel()
+				return fmt.Errorf("waiting for %s (%s): %w", addr, c.Name, err)
+			}
+		}
+	}
+
+	cmd := cmds[len(cmds)-1]
+
+	c := exec.CommandContext(runCtx, cmd.Path, cmd.Args...)
+	c.Env = cmd.Env
+	c.Dir = s.cfg.WorkDir
+	// Its own process group, so stopping kills the engine's children too.
+	// Engines spawn workers; signalling only the parent leaves those workers
+	// holding GPU memory.
+	c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	// CommandContext kills only the leader, so replace that with a group kill.
+	c.Cancel = func() error { return killGroup(c.Process.Pid, syscall.SIGKILL) }
+
+	stdout, err := c.StdoutPipe()
+	if err != nil {
+		cancel()
+		return err
+	}
+	c.Stderr = c.Stdout.(io.Writer) // interleave, as the engine writes them
+
+	if err := c.Start(); err != nil {
+		cancel()
+		return fmt.Errorf("starting %s: %w", cmd.Path, err)
+	}
+
+	sh.mu.Lock()
+	sh.cmd = c
+	sh.pid = c.Process.Pid
+	sh.state = Starting
+	sh.mu.Unlock()
+
+	if err := s.writeState(sh); err != nil {
+		s.log.Warn("could not write pid file; adoption after a restart will not work",
+			"shard", sh.ID, "err", err)
+	}
+
+	go s.pump(sh, stdout, cmd.Name)
+	go s.wait(sh)
+	return nil
+}
+
+// spawn starts one process in its own group and pipes its output to the log.
+func (s *Supervisor) spawn(ctx context.Context, sh *Shard, c engines.Command) (*exec.Cmd, error) {
+	proc := exec.CommandContext(ctx, c.Path, c.Args...)
+	proc.Env = c.Env
+	proc.Dir = s.cfg.WorkDir
+	proc.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	proc.Cancel = func() error { return killGroup(proc.Process.Pid, syscall.SIGKILL) }
+
+	stdout, err := proc.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+	proc.Stderr = proc.Stdout.(io.Writer)
+
+	if err := proc.Start(); err != nil {
+		return nil, fmt.Errorf("starting %s: %w", c.Path, err)
+	}
+	go s.pump(sh, stdout, c.Name)
+	// Reaped so it does not become a zombie; its exit does not end the shard.
+	go func() { _ = proc.Wait() }()
+	return proc, nil
+}
+
+// waitForPort blocks until addr accepts a connection.
+func waitForPort(ctx context.Context, addr string, timeout time.Duration) error {
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	var d net.Dialer
+	for {
+		conn, err := d.DialContext(ctx, "tcp", addr)
+		if err == nil {
+			_ = conn.Close()
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("not listening after %v: %w", timeout, ctx.Err())
+		case <-time.After(200 * time.Millisecond):
+		}
+	}
+}
+
+// pump copies a process's output into the ring buffer and the log.
+func (s *Supervisor) pump(sh *Shard, r io.Reader, name string) {
+	sc := bufio.NewScanner(r)
+	// Engine tracebacks and progress bars produce long lines; the default
+	// 64 KiB limit truncates them exactly when they matter.
+	sc.Buffer(make([]byte, 0, 64<<10), 1<<20)
+
+	for sc.Scan() {
+		line := sc.Text()
+		if name != "" {
+			// Prefixed so the two processes of a Ray shard stay distinguishable
+			// in a single log tail.
+			line = "[" + name + "] " + line
+		}
+		sh.logs.add(line)
+		s.log.Debug("engine", "shard", sh.ID, "line", line)
+	}
+}
+
+// wait reaps the process and records why it ended.
+func (s *Supervisor) wait(sh *Shard) {
+	defer close(sh.done)
+
+	err := sh.cmd.Wait()
+
+	sh.mu.Lock()
+	prev := sh.state
+	if ee := new(exec.ExitError); errors.As(err, &ee) {
+		sh.exitCode = ee.ExitCode()
+	}
+	switch {
+	case prev == Draining || prev == Stopped:
+		// Expected: we asked it to stop.
+		sh.state = Stopped
+	case err == nil:
+		// An engine exiting on its own is a failure, not a success: it is
+		// supposed to serve until told otherwise.
+		sh.state = Failed
+		sh.err = errors.New("engine exited unexpectedly")
+	default:
+		sh.state = Failed
+		sh.err = err
+	}
+	final, code := sh.state, sh.exitCode
+	sh.mu.Unlock()
+
+	s.removeState(sh)
+	if final == Failed {
+		s.log.Error("shard failed", "shard", sh.ID, "exit_code", code, "err", err,
+			"last_output", strings.Join(sh.Logs(5), " | "))
+	} else {
+		s.log.Info("shard stopped", "shard", sh.ID, "exit_code", code)
+	}
+}
+
+// WaitReady polls until the shard serves, it fails, or the timeout expires.
+//
+// The timeout comes from the profile because it is model-dependent: loading a
+// 70B model across two nodes legitimately takes tens of minutes, while a small
+// model failing to load for ten minutes means something is wrong.
+func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model string, timeout time.Duration) error {
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	t := time.NewTicker(s.cfg.HealthInterval)
+	defer t.Stop()
+
+	sh.setState(Loading, nil)
+	for {
+		select {
+		case <-sh.done:
+			return fmt.Errorf("shard %s exited during startup: %s", sh.ID, sh.Status().Err)
+		case <-ctx.Done():
+			last := strings.Join(sh.Logs(10), "\n")
+			return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last)
+		case <-t.C:
+			if err := s.health.Ready(ctx, baseURL, model); err != nil {
+				continue
+			}
+			sh.setState(Ready, nil)
+			return nil
+		}
+	}
+}
+
+// WaitAlive is readiness for a rank that serves no HTTP.
+//
+// A worker in a multi-node replica has no endpoint to poll, so the only
+// evidence available is that it started and stayed started. That is weaker than
+// a health check, and deliberately so: the replica's real readiness is the
+// leader answering, which only happens once every worker has joined. Polling a
+// worker's port instead would time out and fail a replica that is coming up
+// perfectly well.
+func (s *Supervisor) WaitAlive(ctx context.Context, sh *Shard) error {
+	sh.setState(Loading, nil)
+
+	select {
+	case <-sh.done:
+		return fmt.Errorf("shard %s exited during startup: %s", sh.ID, sh.Status().Err)
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-time.After(s.cfg.AliveGrace):
+	}
+
+	if st := sh.Status().State; st.Terminal() {
+		return fmt.Errorf("shard %s is %s: %s", sh.ID, st, sh.Status().Err)
+	}
+	sh.setState(Ready, nil)
+	return nil
+}
+
+// Drain marks a shard as taking no new work. The router stops routing to it;
+// the process keeps running until Stop.
+func (s *Supervisor) Drain(id string) error {
+	sh, err := s.Get(id)
+	if err != nil {
+		return err
+	}
+	if st := sh.Status().State; !st.Live() {
+		return fmt.Errorf("shard %s is %s, cannot drain", id, st)
+	}
+	sh.setState(Draining, nil)
+	return nil
+}
+
+// Stop terminates a shard: SIGTERM to the whole group, then SIGKILL after
+// StopTimeout.
+func (s *Supervisor) Stop(ctx context.Context, id string) error {
+	sh, err := s.Get(id)
+	if err != nil {
+		return err
+	}
+
+	sh.mu.Lock()
+	pid := sh.pid
+	sidecars := slices.Clone(sh.sidecars)
+	if sh.state.Live() {
+		sh.state = Draining // marks the exit as expected for wait() and poll()
+	}
+	sh.mu.Unlock()
+
+	// Sidecars go too. A Ray head left behind holds the port the next replica
+	// needs, and would let a later shard join a cluster that serves nothing.
+	defer func() {
+		for _, sc := range sidecars {
+			if sc.Process == nil {
+				continue
+			}
+			if err := killGroup(sc.Process.Pid, syscall.SIGTERM); err != nil &&
+				!errors.Is(err, os.ErrProcessDone) {
+				s.log.Warn("stopping a sidecar failed", "shard", id, "pid", sc.Process.Pid, "err", err)
+			}
+		}
+	}()
+
+	// A pid of zero means the process never started; an adopted shard has a pid
+	// but no exec.Cmd, and must still be killed by signal.
+	if pid == 0 {
+		sh.setState(Stopped, nil)
+		return nil
+	}
+
+	// Signal the group, not the process: engines fork workers that hold GPU
+	// memory, and those outlive a signal sent only to the leader.
+	if err := killGroup(pid, syscall.SIGTERM); err != nil && !errors.Is(err, os.ErrProcessDone) {
+		s.log.Warn("SIGTERM failed", "shard", id, "err", err)
+	}
+
+	// force kills the group outright. exec.Cmd's cancel does this for shards we
+	// started; an adopted shard has none, so signal its group directly.
+	force := func() {
+		if sh.cancel != nil {
+			sh.cancel()
+			return
+		}
+		if err := killGroup(pid, syscall.SIGKILL); err != nil && !errors.Is(err, os.ErrProcessDone) {
+			s.log.Warn("SIGKILL failed", "shard", id, "err", err)
+		}
+	}
+
+	select {
+	case <-sh.done:
+		return nil
+	case <-time.After(s.cfg.StopTimeout):
+		s.log.Warn("shard did not exit in time, killing", "shard", id, "timeout", s.cfg.StopTimeout)
+		force()
+	case <-ctx.Done():
+		force()
+		return ctx.Err()
+	}
+
+	select {
+	case <-sh.done:
+	case <-time.After(5 * time.Second):
+		return fmt.Errorf("shard %s survived SIGKILL", id)
+	}
+	return nil
+}
+
+// StopAll stops every live shard.
+func (s *Supervisor) StopAll(ctx context.Context) {
+	for _, id := range s.IDs() {
+		if err := s.Stop(ctx, id); err != nil {
+			s.log.Warn("stopping shard failed", "shard", id, "err", err)
+		}
+	}
+}
+
+// Get returns a supervised shard.
+func (s *Supervisor) Get(id string) (*Shard, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	sh, ok := s.shards[id]
+	if !ok {
+		return nil, fmt.Errorf("unknown shard %q", id)
+	}
+	return sh, nil
+}
+
+// Remove forgets a terminated shard.
+func (s *Supervisor) Remove(id string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	delete(s.shards, id)
+}
+
+// IDs lists the supervised shards.
+func (s *Supervisor) IDs() []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	out := make([]string, 0, len(s.shards))
+	for id := range s.shards {
+		out = append(out, id)
+	}
+	return out
+}
+
+// Statuses returns every shard's status.
+func (s *Supervisor) Statuses() []Status {
+	s.mu.Lock()
+	shards := make([]*Shard, 0, len(s.shards))
+	for _, sh := range s.shards {
+		shards = append(shards, sh)
+	}
+	s.mu.Unlock()
+
+	out := make([]Status, 0, len(shards))
+	for _, sh := range shards {
+		out = append(out, sh.Status())
+	}
+	return out
+}
+
+// --- state files, for adoption after an agent restart ---
+
+func (s *Supervisor) statePath(id string) string {
+	return filepath.Join(s.cfg.WorkDir, "shard-"+id+".json")
+}
+
+// shardState is what survives an agent restart.
+//
+// The pid alone is not enough for two separate reasons. Pids are reused, so the
+// command line is kept as a fingerprint — adopting an unrelated process would
+// be worse than leaking the original. And the full spec is kept because a
+// recovered shard has to be described back to the control plane: without it the
+// control plane cannot tell which replica the process belongs to, and would
+// place a duplicate while the original keeps holding the GPUs (plan §6.5).
+type shardState struct {
+	ShardID   string            `json:"shard_id"`
+	ReplicaID string            `json:"replica_id"`
+	PID       int               `json:"pid"`
+	Cmdline   []string          `json:"cmdline"`
+	StartedAt time.Time         `json:"started_at"`
+	Spec      engines.ShardSpec `json:"spec"`
+}
+
+// writeState records everything needed to recognise and describe the process later.
+func (s *Supervisor) writeState(sh *Shard) error {
+	b, err := json.Marshal(shardState{
+		ShardID:   sh.ID,
+		ReplicaID: sh.Spec.ReplicaID,
+		PID:       sh.pid,
+		Cmdline:   sh.cmd.Args,
+		StartedAt: sh.started,
+		Spec:      sh.Spec,
+	})
+	if err != nil {
+		return err
+	}
+	// Written atomically: a half-written state file after a crash would be
+	// indistinguishable from a corrupt one, and the shard would be killed.
+	tmp := s.statePath(sh.ID) + ".tmp"
+	if err := os.WriteFile(tmp, b, 0o600); err != nil {
+		return err
+	}
+	return os.Rename(tmp, s.statePath(sh.ID))
+}
+
+func (s *Supervisor) removeState(sh *Shard) {
+	_ = os.Remove(s.statePath(sh.ID))
+}
+
+// Orphan is an engine process left behind by a previous agent.
+type Orphan struct {
+	ShardID   string
+	ReplicaID string
+	PID       int
+	Cmdline   []string
+	Spec      engines.ShardSpec
+	StartedAt time.Time
+	// Alive reports whether the process still exists.
+	Alive bool
+}
+
+// FindOrphans scans the work directory for processes a previous agent started.
+func (s *Supervisor) FindOrphans() ([]Orphan, error) {
+	entries, err := os.ReadDir(s.cfg.WorkDir)
+	if err != nil {
+		return nil, err
+	}
+
+	var out []Orphan
+	for _, e := range entries {
+		name := e.Name()
+		if !strings.HasPrefix(name, "shard-") || !strings.HasSuffix(name, ".json") {
+			continue
+		}
+
+		b, err := os.ReadFile(filepath.Join(s.cfg.WorkDir, name))
+		if err != nil {
+			continue
+		}
+		var st shardState
+		if err := json.Unmarshal(b, &st); err != nil || st.ShardID == "" || st.PID <= 0 {
+			// A corrupt state file describes a process that cannot be
+			// identified, so it cannot be adopted safely. Leave it for
+			// KillOrphan to clean up by shard id from the filename.
+			id := strings.TrimSuffix(strings.TrimPrefix(name, "shard-"), ".json")
+			out = append(out, Orphan{ShardID: id})
+			continue
+		}
+
+		out = append(out, Orphan{
+			ShardID:   st.ShardID,
+			ReplicaID: st.ReplicaID,
+			PID:       st.PID,
+			Cmdline:   st.Cmdline,
+			Spec:      st.Spec,
+			StartedAt: st.StartedAt,
+			Alive:     processAlive(st.PID),
+		})
+	}
+	return out, nil
+}
+
+// Adopt takes over a still-running engine from a previous agent.
+//
+// The process cannot be waited on — it is not this process's child — so it is
+// watched by polling instead. That is the whole difference between an adopted
+// shard and a started one; everything downstream treats them the same.
+func (s *Supervisor) Adopt(o Orphan) (*Shard, error) {
+	if !o.Alive {
+		return nil, fmt.Errorf("shard %s: process %d is gone", o.ShardID, o.PID)
+	}
+
+	s.mu.Lock()
+	if _, exists := s.shards[o.ShardID]; exists {
+		s.mu.Unlock()
+		return nil, ErrAlreadyRunning
+	}
+	sh := &Shard{
+		ID:      o.ShardID,
+		Spec:    o.Spec,
+		state:   Starting,
+		pid:     o.PID,
+		logs:    newRingBuffer(s.cfg.LogLines),
+		started: o.StartedAt,
+		done:    make(chan struct{}),
+		adopted: true,
+	}
+	s.shards[o.ShardID] = sh
+	s.mu.Unlock()
+
+	// Output is gone with the previous agent; say so rather than presenting an
+	// empty log as if the engine had been silent.
+	sh.logs.add("[adopted from a previous agent; earlier output is not available]")
+
+	go s.poll(sh)
+	return sh, nil
+}
+
+// poll watches an adopted process, which cannot be reaped with Wait.
+func (s *Supervisor) poll(sh *Shard) {
+	defer close(sh.done)
+
+	// Polled rather than waited on, so the interval bounds how quickly a stop
+	// is observed. Short enough that Stop does not sit out its whole timeout.
+	t := time.NewTicker(200 * time.Millisecond)
+	defer t.Stop()
+	for range t.C {
+		if processAlive(sh.pid) {
+			continue
+		}
+		sh.mu.Lock()
+		expected := sh.state == Draining || sh.state == Stopped
+		if expected {
+			sh.state = Stopped
+		} else {
+			sh.state = Failed
+			sh.err = errors.New("adopted engine exited")
+		}
+		sh.mu.Unlock()
+
+		s.removeState(sh)
+		s.log.Info("adopted shard exited", "shard", sh.ID, "expected", expected)
+		return
+	}
+}
+
+// KillOrphan terminates a leftover process group and removes its pid file.
+func (s *Supervisor) KillOrphan(o Orphan) error {
+	if o.Alive {
+		if err := killGroup(o.PID, syscall.SIGTERM); err != nil {
+			return err
+		}
+		// Give it a moment to free GPU memory before insisting.
+		deadline := time.Now().Add(s.cfg.StopTimeout)
+		for time.Now().Before(deadline) {
+			if !processAlive(o.PID) {
+				break
+			}
+			time.Sleep(200 * time.Millisecond)
+		}
+		if processAlive(o.PID) {
+			_ = killGroup(o.PID, syscall.SIGKILL)
+		}
+	}
+	// An already-missing state file is success: the previous agent may have
+	// reaped the process itself between the scan and this call.
+	if err := os.Remove(s.statePath(o.ShardID)); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	return nil
+}

internal/agent/supervisor/supervisor_test.go

@@ -0,0 +1,796 @@
+package supervisor
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync/atomic"
+	"syscall"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+)
+
+// scriptedHealth reports ready once the flag is set.
+type scriptedHealth struct{ ready atomic.Bool }
+
+func (h *scriptedHealth) Ready(context.Context, string, string) error {
+	if h.ready.Load() {
+		return nil
+	}
+	return errors.New("not ready")
+}
+
+func newSupervisor(t *testing.T) (*Supervisor, *scriptedHealth) {
+	t.Helper()
+	h := &scriptedHealth{}
+	s, err := New(Config{
+		WorkDir:        t.TempDir(),
+		LogLines:       64,
+		StopTimeout:    3 * time.Second,
+		HealthInterval: 20 * time.Millisecond,
+	}, h, slog.New(slog.DiscardHandler))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { s.StopAll(context.Background()) })
+	return s, h
+}
+
+// shellCommand builds the single-command sequence for a shell script.
+func shellCommand(script string) []engines.Command {
+	return []engines.Command{{Path: "/bin/sh", Args: []string{"-c", script}, Env: os.Environ()}}
+}
+
+func waitState(t *testing.T, sh *Shard, want State, limit time.Duration) {
+	t.Helper()
+	deadline := time.Now().Add(limit)
+	for time.Now().Before(deadline) {
+		if sh.Status().State == want {
+			return
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	t.Fatalf("shard stayed in %s, expected %s", sh.Status().State, want)
+}
+
+func TestStartAndStop(t *testing.T) {
+	s, _ := newSupervisor(t)
+
+	sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if st := sh.Status(); st.State != Starting || st.PID == 0 {
+		t.Fatalf("status after start: %+v", st)
+	}
+
+	if err := s.Stop(t.Context(), "a"); err != nil {
+		t.Fatal(err)
+	}
+	if st := sh.Status().State; st != Stopped {
+		t.Errorf("state after stop: %s", st)
+	}
+}
+
+func TestUnexpectedExitIsFailure(t *testing.T) {
+	// An engine exiting on its own is a failure even with status 0: it is meant
+	// to serve until told otherwise.
+	s, _ := newSupervisor(t)
+
+	sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 0"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	waitState(t, sh, Failed, 3*time.Second)
+	if !strings.Contains(sh.Status().Err, "unexpectedly") {
+		t.Errorf("err=%q", sh.Status().Err)
+	}
+}
+
+func TestNonZeroExitRecordsCode(t *testing.T) {
+	s, _ := newSupervisor(t)
+
+	sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 42"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	waitState(t, sh, Failed, 3*time.Second)
+	if got := sh.Status().ExitCode; got != 42 {
+		t.Errorf("exit code=%d, expected 42", got)
+	}
+}
+
+func TestStopKillsChildProcesses(t *testing.T) {
+	// The reason for the process group: engines fork workers that hold GPU
+	// memory, and signalling only the leader leaves them behind.
+	s, _ := newSupervisor(t)
+	dir := t.TempDir()
+	marker := filepath.Join(dir, "child.pid")
+
+	// The shell forks a child that writes its own pid, then both sleep.
+	script := fmt.Sprintf(`sh -c 'echo $$ > %s; sleep 30' & sleep 30`, marker)
+	if _, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand(script)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the child to announce itself.
+	var childPID int
+	deadline := time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		if b, err := os.ReadFile(marker); err == nil {
+			if _, err := fmt.Sscanf(strings.TrimSpace(string(b)), "%d", &childPID); err == nil && childPID > 0 {
+				break
+			}
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if childPID == 0 {
+		t.Fatal("child never started")
+	}
+
+	if err := s.Stop(t.Context(), "a"); err != nil {
+		t.Fatal(err)
+	}
+
+	deadline = time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		if !processAlive(childPID) {
+			return
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	t.Errorf("child process %d survived the stop; it would still hold GPU memory", childPID)
+}
+
+func TestStopKillsAfterTimeout(t *testing.T) {
+	// A process ignoring SIGTERM must still die.
+	s, _ := newSupervisor(t)
+	s.cfg.StopTimeout = 300 * time.Millisecond
+
+	sh, err := s.Start(t.Context(), "a", engines.ShardSpec{},
+		shellCommand("trap '' TERM; sleep 30"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	pid := sh.Status().PID
+
+	start := time.Now()
+	if err := s.Stop(t.Context(), "a"); err != nil {
+		t.Fatal(err)
+	}
+	if d := time.Since(start); d > 5*time.Second {
+		t.Errorf("stop took %v", d)
+	}
+	if processAlive(pid) {
+		t.Error("the process survived SIGKILL")
+	}
+}
+
+func TestStartIsIdempotent(t *testing.T) {
+	// The control plane retries commands, so a repeated start must not produce
+	// a second process (plan §5.6).
+	s, _ := newSupervisor(t)
+
+	first, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	second, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
+	if !errors.Is(err, ErrAlreadyRunning) {
+		t.Fatalf("expected ErrAlreadyRunning, got %v", err)
+	}
+	if first.Status().PID != second.Status().PID {
+		t.Error("a second process was started for the same shard")
+	}
+}
+
+func TestRestartAfterTerminalState(t *testing.T) {
+	s, _ := newSupervisor(t)
+
+	sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 1"))
+	waitState(t, sh, Failed, 3*time.Second)
+
+	// Once it has failed, the same id may be started again.
+	again, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
+	if err != nil {
+		t.Fatalf("restarting a failed shard: %v", err)
+	}
+	if again.Status().State != Starting {
+		t.Errorf("state=%s", again.Status().State)
+	}
+}
+
+func TestWaitReady(t *testing.T) {
+	s, h := newSupervisor(t)
+
+	sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	go func() {
+		time.Sleep(100 * time.Millisecond)
+		h.ready.Store(true)
+	}()
+
+	if err := s.WaitReady(t.Context(), sh, "http://x", "m", 3*time.Second); err != nil {
+		t.Fatal(err)
+	}
+	st := sh.Status()
+	if st.State != Ready {
+		t.Errorf("state=%s", st.State)
+	}
+	if st.ReadyAt.IsZero() {
+		t.Error("ReadyAt was not recorded")
+	}
+}
+
+func TestWaitReadyTimeoutIncludesLogs(t *testing.T) {
+	// When a model fails to load, the engine output is the only diagnosis
+	// available, so the error has to carry it.
+	s, _ := newSupervisor(t)
+
+	sh, err := s.Start(t.Context(), "a", engines.ShardSpec{},
+		shellCommand("echo 'CUDA out of memory'; sleep 30"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(100 * time.Millisecond) // let the line be captured
+
+	err = s.WaitReady(t.Context(), sh, "http://x", "m", 200*time.Millisecond)
+	if err == nil {
+		t.Fatal("expected a timeout")
+	}
+	if !strings.Contains(err.Error(), "CUDA out of memory") {
+		t.Errorf("the error does not carry engine output: %v", err)
+	}
+}
+
+func TestWaitReadyDetectsEarlyExit(t *testing.T) {
+	s, _ := newSupervisor(t)
+
+	sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 1"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Should return as soon as the process dies, not after the full timeout.
+	start := time.Now()
+	if err := s.WaitReady(t.Context(), sh, "http://x", "m", 10*time.Second); err == nil {
+		t.Fatal("expected an error")
+	}
+	if d := time.Since(start); d > 3*time.Second {
+		t.Errorf("waited %v for a process that had already exited", d)
+	}
+}
+
+func TestDrain(t *testing.T) {
+	s, _ := newSupervisor(t)
+
+	sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
+	if err := s.Drain("a"); err != nil {
+		t.Fatal(err)
+	}
+	if got := sh.Status().State; got != Draining {
+		t.Errorf("state=%s", got)
+	}
+	// A drained shard's process is still alive; only routing stops.
+	if !processAlive(sh.Status().PID) {
+		t.Error("draining killed the process; it should keep serving in-flight work")
+	}
+}
+
+func TestLogsAreCaptured(t *testing.T) {
+	s, _ := newSupervisor(t)
+
+	sh, err := s.Start(t.Context(), "a", engines.ShardSpec{},
+		shellCommand("echo line1; echo line2 >&2; echo line3; sleep 0.2"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(400 * time.Millisecond)
+
+	logs := strings.Join(sh.Logs(10), "\n")
+	for _, want := range []string{"line1", "line2", "line3"} {
+		if !strings.Contains(logs, want) {
+			t.Errorf("%q missing from logs:\n%s", want, logs)
+		}
+	}
+}
+
+// --- orphan adoption ---
+
+func TestFindOrphans(t *testing.T) {
+	s, _ := newSupervisor(t)
+
+	spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20100}
+	sh, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	pid := sh.Status().PID
+
+	// Simulate an agent restart: a new supervisor over the same work dir.
+	s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
+		&scriptedHealth{}, slog.New(slog.DiscardHandler))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	orphans, err := s2.FindOrphans()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(orphans) != 1 {
+		t.Fatalf("expected 1 orphan, got %d", len(orphans))
+	}
+	o := orphans[0]
+	if o.ShardID != "a" || o.PID != pid || !o.Alive {
+		t.Errorf("orphan=%+v (want shard a, pid %d, alive)", o, pid)
+	}
+	// The command line is stored as a fingerprint, because pids get reused and
+	// adopting an unrelated process is worse than leaking the original.
+	if len(o.Cmdline) == 0 || !strings.Contains(strings.Join(o.Cmdline, " "), "sleep 30") {
+		t.Errorf("cmdline fingerprint missing: %v", o.Cmdline)
+	}
+	// And the spec, without which the control plane could not tell which
+	// replica the process belongs to and would place a duplicate.
+	if o.ReplicaID != "rep-1" {
+		t.Errorf("replica id not recovered: %q", o.ReplicaID)
+	}
+	if o.Spec.HTTPPort != 20100 {
+		t.Errorf("spec not recovered: %+v", o.Spec)
+	}
+
+	if err := s2.KillOrphan(o); err != nil {
+		t.Fatal(err)
+	}
+	deadline := time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		if !processAlive(pid) {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	if processAlive(pid) {
+		t.Error("the orphan survived KillOrphan")
+	}
+	if _, err := os.Stat(s2.statePath("a")); !os.IsNotExist(err) {
+		t.Error("the state file was left behind")
+	}
+}
+
+func TestStateFileRemovedOnExit(t *testing.T) {
+	s, _ := newSupervisor(t)
+
+	sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 0"))
+	waitState(t, sh, Failed, 3*time.Second)
+	time.Sleep(50 * time.Millisecond)
+
+	if _, err := os.Stat(s.statePath("a")); !os.IsNotExist(err) {
+		t.Error("the state file outlived the process; a later scan would find a dead orphan")
+	}
+}
+
+// --- state machine and ring buffer ---
+
+func TestStateStrings(t *testing.T) {
+	for st, want := range map[State]string{
+		Pending: "pending", Starting: "starting", Loading: "loading",
+		Ready: "ready", Draining: "draining", Stopped: "stopped", Failed: "failed",
+	} {
+		if got := st.String(); got != want {
+			t.Errorf("%d -> %q, want %q", int(st), got, want)
+		}
+	}
+	if !Stopped.Terminal() || !Failed.Terminal() || Ready.Terminal() {
+		t.Error("Terminal is wrong")
+	}
+	if !Ready.Live() || !Draining.Live() || Stopped.Live() {
+		t.Error("Live is wrong")
+	}
+}
+
+func TestRingBuffer(t *testing.T) {
+	r := newRingBuffer(3)
+	if got := r.tail(10); len(got) != 0 {
+		t.Errorf("empty buffer returned %v", got)
+	}
+
+	r.add("a")
+	r.add("b")
+	if got := strings.Join(r.tail(10), ","); got != "a,b" {
+		t.Errorf("tail=%q", got)
+	}
+
+	r.add("c")
+	r.add("d") // evicts "a"
+	if got := strings.Join(r.tail(10), ","); got != "b,c,d" {
+		t.Errorf("after wrap tail=%q, want b,c,d", got)
+	}
+	if got := strings.Join(r.tail(2), ","); got != "c,d" {
+		t.Errorf("tail(2)=%q", got)
+	}
+}
+
+func TestAdoptTakesOverRunningShard(t *testing.T) {
+	// The control plane restart fix depends on this: a shard that survived the
+	// agent must come back under supervision, not be restarted or leaked.
+	s, _ := newSupervisor(t)
+
+	spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20200}
+	sh, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	pid := sh.Status().PID
+
+	// A new supervisor over the same work dir, as after an agent restart.
+	s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 3 * time.Second},
+		&scriptedHealth{}, slog.New(slog.DiscardHandler))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	orphans, err := s2.FindOrphans()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(orphans) != 1 || !orphans[0].Alive {
+		t.Fatalf("orphan not found: %+v", orphans)
+	}
+
+	adopted, err := s2.Adopt(orphans[0])
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := adopted.Status().PID; got != pid {
+		t.Errorf("adopted pid %d, expected %d", got, pid)
+	}
+	if adopted.Spec.ReplicaID != "rep-1" {
+		t.Errorf("spec lost in adoption: %+v", adopted.Spec)
+	}
+	// The earlier output is gone; saying so beats presenting an empty log as if
+	// the engine had been silent.
+	if logs := strings.Join(adopted.Logs(5), " "); !strings.Contains(logs, "adopted") {
+		t.Errorf("adoption was not noted in the log: %q", logs)
+	}
+
+	// Stopping an adopted shard must actually kill it: it has no exec.Cmd, so
+	// the signal path is the only one available.
+	if err := s2.Stop(t.Context(), "a"); err != nil {
+		t.Fatal(err)
+	}
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		if !processAlive(pid) {
+			return
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	t.Error("the adopted process survived Stop")
+}
+
+func TestAdoptRefusesDeadProcess(t *testing.T) {
+	s, _ := newSupervisor(t)
+	if _, err := s.Adopt(Orphan{ShardID: "gone", PID: 999999, Alive: false}); err == nil {
+		t.Error("a dead process was adopted")
+	}
+}
+
+func TestAdoptedExitIsReported(t *testing.T) {
+	// An adopted engine dying has to be noticed; nothing waits on it.
+	s, _ := newSupervisor(t)
+
+	spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a"}
+	sh, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	pid := sh.Status().PID
+
+	s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
+		&scriptedHealth{}, slog.New(slog.DiscardHandler))
+	if err != nil {
+		t.Fatal(err)
+	}
+	orphans, _ := s2.FindOrphans()
+	adopted, err := s2.Adopt(orphans[0])
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Kill it behind the supervisor's back.
+	_ = killGroup(pid, syscall.SIGKILL)
+	waitState(t, adopted, Failed, 5*time.Second)
+}
+
+func TestCorruptStateFileIsCleanedUp(t *testing.T) {
+	// A half-written state file describes a process that cannot be identified,
+	// so it cannot be adopted safely.
+	s, _ := newSupervisor(t)
+	if err := os.WriteFile(s.statePath("broken"), []byte("{not json"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	orphans, err := s.FindOrphans()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(orphans) != 1 || orphans[0].ShardID != "broken" || orphans[0].Alive {
+		t.Fatalf("corrupt state file not reported for cleanup: %+v", orphans)
+	}
+	if err := s.KillOrphan(orphans[0]); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(s.statePath("broken")); !os.IsNotExist(err) {
+		t.Error("the corrupt state file was left behind")
+	}
+}
+
+// --- multi-process shards (plan §11, phase 4) ---
+
+func TestASidecarAndAnEngineBothRunUnderOneShard(t *testing.T) {
+	// The Ray shape: a supporting process runs alongside the engine for the
+	// shard's lifetime, but the engine is the one whose exit ends the shard.
+	s, _ := newSupervisor(t)
+	dir := t.TempDir()
+	order := filepath.Join(dir, "order")
+
+	// Each writes to the order file and to stdout: the file records the
+	// sequence, stdout exercises per-process log attribution.
+	cmds := []engines.Command{
+		{Name: "sidecar", Path: "/bin/sh", Args: []string{"-c",
+			fmt.Sprintf("echo sidecar >> %s; echo hello-from-sidecar; sleep 30", order)}, Env: os.Environ()},
+		{Name: "engine", Path: "/bin/sh", Args: []string{"-c",
+			fmt.Sprintf("echo engine >> %s; echo hello-from-engine; sleep 30", order)}, Env: os.Environ()},
+	}
+
+	sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, cmds)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Generous, because this waits on two processes starting under the race
+	// detector while every other package is competing for the same cores. Three
+	// seconds was enough alone and not in a full run.
+	var started []string
+	deadline := time.Now().Add(20 * time.Second)
+	for time.Now().Before(deadline) && len(started) < 2 {
+		if b, err := os.ReadFile(order); err == nil {
+			started = strings.Fields(string(b))
+		}
+		if len(started) < 2 {
+			time.Sleep(20 * time.Millisecond)
+		}
+	}
+	// Asserted rather than broken out of: the original fell through on timeout,
+	// so a slow machine turned this into a no-op and the failure surfaced later
+	// as something else.
+	if len(started) != 2 {
+		t.Fatalf("only %d of 2 processes started: %v", len(started), started)
+	}
+	// Which of the two appended first is deliberately not asserted. The
+	// supervisor spawns the sidecar first, but spawning is not sequencing:
+	// nothing here makes the engine wait, so two shells racing to append give
+	// either order and a test that demanded one would fail for the scheduler's
+	// reasons rather than the supervisor's. The wait that is a real guarantee
+	// is WaitForLeaderPort, and TestWaitForLeaderPortBlocksUntilListening is
+	// where it is checked.
+	if !slices.Contains(started, "sidecar") || !slices.Contains(started, "engine") {
+		t.Fatalf("expected both processes to have run, got %v", started)
+	}
+
+	// The shard's pid is the engine's, not the sidecar's: the engine's exit is
+	// what ends the shard.
+	if sh.Status().PID == 0 {
+		t.Fatal("no engine pid recorded")
+	}
+	// Both processes appear in one log tail, distinguishable by prefix.
+	deadline = time.Now().Add(20 * time.Second)
+	for time.Now().Before(deadline) {
+		logs := strings.Join(sh.Logs(20), "\n")
+		if strings.Contains(logs, "[sidecar] hello-from-sidecar") &&
+			strings.Contains(logs, "[engine] hello-from-engine") {
+			return
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	t.Errorf("output is not attributed per process:\n%s", strings.Join(sh.Logs(20), "\n"))
+}
+
+func TestStopKillsSidecar(t *testing.T) {
+	// A Ray head left behind holds the port the next replica needs, and would
+	// let a later shard join a cluster that serves nothing.
+	s, _ := newSupervisor(t)
+	dir := t.TempDir()
+	marker := filepath.Join(dir, "sidecar.pid")
+
+	cmds := []engines.Command{
+		{Name: "sidecar", Path: "/bin/sh", Args: []string{"-c",
+			fmt.Sprintf("echo $$ > %s; sleep 30", marker)}, Env: os.Environ()},
+		{Name: "engine", Path: "/bin/sh", Args: []string{"-c", "sleep 30"}, Env: os.Environ()},
+	}
+	if _, err := s.Start(t.Context(), "a", engines.ShardSpec{}, cmds); err != nil {
+		t.Fatal(err)
+	}
+
+	var sidePID int
+	deadline := time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		if b, err := os.ReadFile(marker); err == nil {
+			if _, err := fmt.Sscanf(strings.TrimSpace(string(b)), "%d", &sidePID); err == nil && sidePID > 0 {
+				break
+			}
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if sidePID == 0 {
+		t.Fatal("the sidecar never started")
+	}
+
+	if err := s.Stop(t.Context(), "a"); err != nil {
+		t.Fatal(err)
+	}
+	deadline = time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		if !processAlive(sidePID) {
+			return
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	t.Errorf("the sidecar (%d) survived the stop and still holds its port", sidePID)
+}
+
+func TestSidecarFailureFailsTheShard(t *testing.T) {
+	s, _ := newSupervisor(t)
+	cmds := []engines.Command{
+		{Name: "sidecar", Path: "/nonexistent/binary", Env: os.Environ()},
+		{Name: "engine", Path: "/bin/sh", Args: []string{"-c", "sleep 30"}, Env: os.Environ()},
+	}
+	sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, cmds)
+	if err == nil {
+		t.Fatal("a shard whose sidecar could not start was reported healthy")
+	}
+	if !strings.Contains(err.Error(), "sidecar") {
+		t.Errorf("the error does not identify the sidecar: %v", err)
+	}
+	if sh.Status().State != Failed {
+		t.Errorf("state=%s", sh.Status().State)
+	}
+}
+
+func TestWaitForLeaderPortBlocksUntilListening(t *testing.T) {
+	// A Ray worker cannot join a head that is not up, and the leader's server
+	// cannot claim GPUs from a cluster that does not exist yet.
+	s, _ := newSupervisor(t)
+	s.cfg.RendezvousTimeout = 5 * time.Second
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	port := ln.Addr().(*net.TCPAddr).Port
+	_ = ln.Close() // nothing listening yet
+
+	spec := engines.ShardSpec{LeaderAddr: "127.0.0.1", LeaderPort: port}
+	cmds := []engines.Command{
+		{Name: "sidecar", Path: "/bin/sh", Args: []string{"-c",
+			fmt.Sprintf("sleep 0.4; nc -l %d", port)}, Env: os.Environ(),
+			WaitForLeaderPort: true},
+		{Name: "engine", Path: "/bin/sh", Args: []string{"-c", "sleep 30"}, Env: os.Environ()},
+	}
+
+	start := time.Now()
+	if _, err := s.Start(t.Context(), "a", spec, cmds); err != nil {
+		t.Skipf("nc is unavailable for the rendezvous stub: %v", err)
+	}
+	if d := time.Since(start); d < 300*time.Millisecond {
+		t.Errorf("the engine started after %v, before the rendezvous point was listening", d)
+	}
+}
+
+func TestRendezvousTimeout(t *testing.T) {
+	// Nothing ever listens, so the shard must fail rather than hang forever.
+	s, _ := newSupervisor(t)
+	s.cfg.RendezvousTimeout = 300 * time.Millisecond
+
+	spec := engines.ShardSpec{LeaderAddr: "127.0.0.1", LeaderPort: 1}
+	cmds := []engines.Command{
+		{Name: "sidecar", Path: "/bin/sh", Args: []string{"-c", "sleep 30"},
+			Env: os.Environ(), WaitForLeaderPort: true},
+		{Name: "engine", Path: "/bin/sh", Args: []string{"-c", "sleep 30"}, Env: os.Environ()},
+	}
+	if _, err := s.Start(t.Context(), "a", spec, cmds); err == nil {
+		t.Fatal("the shard did not fail when the rendezvous point never appeared")
+	}
+}
+
+func TestWaitAliveForNonServingRank(t *testing.T) {
+	// A worker has no endpoint to poll, so staying up is the only evidence
+	// available. This is what stopped multi-node replicas coming up at all.
+	s, _ := newSupervisor(t)
+	s.cfg.AliveGrace = 150 * time.Millisecond
+
+	sh, err := s.Start(t.Context(), "a", engines.ShardSpec{Rank: 1}, shellCommand("sleep 30"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := s.WaitAlive(t.Context(), sh); err != nil {
+		t.Fatal(err)
+	}
+	if got := sh.Status().State; got != Ready {
+		t.Errorf("state=%s", got)
+	}
+}
+
+func TestWaitAliveFailsOnEarlyExit(t *testing.T) {
+	s, _ := newSupervisor(t)
+	s.cfg.AliveGrace = 2 * time.Second
+
+	sh, err := s.Start(t.Context(), "a", engines.ShardSpec{Rank: 1}, shellCommand("exit 1"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	start := time.Now()
+	if err := s.WaitAlive(t.Context(), sh); err == nil {
+		t.Fatal("a worker that died immediately was reported ready")
+	}
+	if d := time.Since(start); d > 1500*time.Millisecond {
+		t.Errorf("waited %v for a process that had already exited", d)
+	}
+}
+
+func TestAnAdoptedShardCanBecomeReady(t *testing.T) {
+	// Adoption stopped at "the process is alive", leaving the shard in Starting
+	// with nothing that would ever move it. The control plane then waited out
+	// the startup timeout and killed a perfectly healthy engine to start an
+	// identical one — so an agent restart, which exists precisely so engines
+	// are not interrupted, cost every model on the node a full reload.
+	//
+	// Seen on node05: the engine answered /health throughout and the replica
+	// sat in Starting for three minutes until the timeout reaped it.
+	s, _ := newSupervisor(t)
+	spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20201}
+	if _, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30")); err != nil {
+		t.Fatal(err)
+	}
+
+	health := &scriptedHealth{}
+	s2, err := New(Config{
+		WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second,
+		HealthInterval: 10 * time.Millisecond,
+	}, health, slog.New(slog.DiscardHandler))
+	if err != nil {
+		t.Fatal(err)
+	}
+	orphans, err := s2.FindOrphans()
+	if err != nil || len(orphans) != 1 {
+		t.Fatalf("orphan not found: %+v (%v)", orphans, err)
+	}
+	adopted, err := s2.Adopt(orphans[0])
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := adopted.Status().State; got != Starting {
+		t.Fatalf("a freshly adopted shard is %s, expected Starting", got)
+	}
+
+	// The engine answers, as node05's did. The shard must be able to reach
+	// Ready from here; before this it had no path to it at all.
+	health.ready.Store(true)
+	if err := s2.WaitReady(t.Context(), adopted, "http://127.0.0.1:20201", "m", 2*time.Second); err != nil {
+		t.Fatalf("an adopted, healthy engine never became ready: %v", err)
+	}
+	if got := adopted.Status().State; got != Ready {
+		t.Errorf("state %s after a passing health check", got)
+	}
+	_ = s2.Stop(t.Context(), "a")
+}

internal/agent/weights/fetch.go

@@ -0,0 +1,323 @@
+// Package weights makes a model's files present on a node.
+//
+// The simple arrangement, and the default: every node has its own copy. A
+// source that is already a local path is used as it stands; one that names a
+// remote file is fetched into a cache directory once and reused. Nothing here
+// manages storage, decides a retention policy, or knows what a filesystem is —
+// it answers one question, which is where the engine should be pointed.
+//
+// The exotic arrangements are all handled by not acting. Weights on a shared
+// filer, an image exported over the fabric, or anything else an operator has
+// mounted are simply present, and present means there is nothing to do. That is
+// why the interesting case needs no code: the escape hatch from a distribution
+// mechanism is a distribution mechanism that checks first.
+package weights
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Progress reports how a fetch is going, for anything that wants to say so.
+type Progress struct {
+	Source     string
+	BytesDone  int64
+	BytesTotal int64
+}
+
+// Percent is how far along, or -1 when the total is unknown.
+func (p Progress) Percent() int {
+	if p.BytesTotal <= 0 {
+		return -1
+	}
+	return int(p.BytesDone * 100 / p.BytesTotal)
+}
+
+// Fetcher makes sources local.
+type Fetcher struct {
+	CacheDir string
+	Client   *http.Client
+	Log      *slog.Logger
+	// OnProgress is called occasionally during a fetch. A missing model is the
+	// difference between a replica that is slow to start and one that is stuck,
+	// and without this the two are indistinguishable from outside.
+	OnProgress func(Progress)
+
+	// inflight makes concurrent requests for one source share a single fetch.
+	//
+	// Eight shards of a tensor-parallel replica start at the same instant and
+	// want the same file. Without this they would each open a connection and
+	// each write the same bytes, turning one download into eight and, on a
+	// shared origin, into a self-inflicted denial of service.
+	mu       sync.Mutex
+	inflight map[string]*fetchCall
+}
+
+type fetchCall struct {
+	done chan struct{}
+	path string
+	err  error
+}
+
+// ErrNotLocal means a source names a file that is not here and cannot be
+// fetched — a bare path with nothing to fetch it from.
+var ErrNotLocal = errors.New("the weights are not on this node and the source says nothing about where to get them")
+
+// Ensure returns a local path for a source, fetching it if it is not here.
+//
+// A plain path is never fetched, only checked. That is deliberate: a path is
+// what an operator writes when the file is already there, whether it arrived on
+// local disk, a Lustre mount or an image exported over the fabric, and a
+// downloader that second-guessed that would copy a hundred gigabytes that were
+// already present.
+func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
+	if source == "" {
+		return "", fmt.Errorf("no source given")
+	}
+	// A version rather than a file. Everything below assumes one URL is one
+	// file, and a set has to be handled before any of it.
+	if IsSet(source) {
+		return f.ensureSet(ctx, source)
+	}
+	if !isRemote(source) {
+		if !isPath(source) {
+			// A bare model id — "Qwen/Qwen2.5-7B-Instruct" — is not a file and
+			// was never meant to be one. vLLM and SGLang resolve those
+			// themselves, and doing it for them would be both a duplicate
+			// downloader and a change in documented behaviour.
+			return source, nil
+		}
+		if _, err := os.Stat(source); err == nil {
+			return source, nil
+		}
+		return "", fmt.Errorf("%w: %s", ErrNotLocal, source)
+	}
+
+	target, err := f.cachePath(source)
+	if err != nil {
+		return "", err
+	}
+	if _, err := os.Stat(target); err == nil {
+		return target, nil
+	}
+
+	f.mu.Lock()
+	if f.inflight == nil {
+		f.inflight = map[string]*fetchCall{}
+	}
+	if call, running := f.inflight[source]; running {
+		f.mu.Unlock()
+		select {
+		case <-call.done:
+			return call.path, call.err
+		case <-ctx.Done():
+			return "", ctx.Err()
+		}
+	}
+	call := &fetchCall{done: make(chan struct{})}
+	f.inflight[source] = call
+	f.mu.Unlock()
+
+	call.path, call.err = f.fetch(ctx, source, target)
+	close(call.done)
+
+	f.mu.Lock()
+	delete(f.inflight, source)
+	f.mu.Unlock()
+	return call.path, call.err
+}
+
+// fetch downloads a source to a target, resuming and verifying.
+func (f *Fetcher) fetch(ctx context.Context, source, target string) (string, error) {
+	return f.fetchAs(ctx, source, source, target)
+}
+
+// fetchAs downloads one URL but reports progress under another name.
+//
+// The two differ for a set: the shards waiting on it asked for the version, not
+// for the fourth file in it, and progress reported under the file's own URL
+// reaches nobody. What they see is one number climbing across the whole set.
+func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string) (string, error) {
+	url, err := resolveURL(source)
+	if err != nil {
+		return "", err
+	}
+	if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+		return "", err
+	}
+
+	// Written beside the target and renamed at the end. A partial file under
+	// the real name would be found by the next scan, reported as present, and
+	// handed to an engine that fails on it minutes later — the exact failure
+	// this whole area exists to prevent.
+	part := target + ".part"
+	existing := int64(0)
+	if st, err := os.Stat(part); err == nil {
+		existing = st.Size()
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return "", err
+	}
+	if existing > 0 {
+		// Resume. A forty gigabyte fetch that fails at ninety percent must not
+		// start again from nothing.
+		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", existing))
+	}
+
+	client := f.Client
+	if client == nil {
+		client = &http.Client{}
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("fetching %s: %w", source, err)
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		existing = 0 // the server ignored the range; start over
+	case http.StatusPartialContent:
+	default:
+		return "", fmt.Errorf("fetching %s: %s returned %s", source, url, resp.Status)
+	}
+
+	flags := os.O_CREATE | os.O_WRONLY
+	if existing > 0 {
+		flags |= os.O_APPEND
+	} else {
+		flags |= os.O_TRUNC
+	}
+	out, err := os.OpenFile(part, flags, 0o644)
+	if err != nil {
+		return "", err
+	}
+
+	total := existing + resp.ContentLength
+	f.report(Progress{Source: reportAs, BytesDone: existing, BytesTotal: total})
+	written, err := f.copyWithProgress(ctx, out, resp.Body, reportAs, existing, total)
+	closeErr := out.Close()
+	if err != nil {
+		// The part file is kept on purpose, so the next attempt resumes.
+		return "", fmt.Errorf("fetching %s after %d bytes: %w", source, written, err)
+	}
+	if closeErr != nil {
+		return "", closeErr
+	}
+
+	if err := os.Rename(part, target); err != nil {
+		return "", err
+	}
+	if f.Log != nil {
+		f.Log.Info("weights fetched", "source", source, "path", target, "bytes", written)
+	}
+	return target, nil
+}
+
+func (f *Fetcher) copyWithProgress(ctx context.Context, dst io.Writer, src io.Reader,
+	source string, done, total int64,
+) (int64, error) {
+	buf := make([]byte, 4<<20)
+	last := time.Now()
+	for {
+		if err := ctx.Err(); err != nil {
+			return done, err
+		}
+		n, err := src.Read(buf)
+		if n > 0 {
+			if _, werr := dst.Write(buf[:n]); werr != nil {
+				return done, werr
+			}
+			done += int64(n)
+			// Occasionally, not per read: a four-megabyte buffer over a
+			// hundred gigabytes is twenty-five thousand callbacks, and nobody
+			// needs to be told that often.
+			if time.Since(last) > 2*time.Second {
+				f.report(Progress{Source: source, BytesDone: done, BytesTotal: total})
+				last = time.Now()
+			}
+		}
+		if errors.Is(err, io.EOF) {
+			f.report(Progress{Source: source, BytesDone: done, BytesTotal: total})
+			return done, nil
+		}
+		if err != nil {
+			return done, err
+		}
+	}
+}
+
+func (f *Fetcher) report(p Progress) {
+	if f.OnProgress != nil {
+		f.OnProgress(p)
+	}
+}
+
+// cachePath is where a remote source lands.
+//
+// Named by a hash of the source with the original filename kept on the end, so
+// two models with the same filename from different places cannot collide while
+// a human looking at the directory can still tell what is in it.
+func (f *Fetcher) cachePath(source string) (string, error) {
+	if f.CacheDir == "" {
+		return "", fmt.Errorf("no cache directory configured, so %s cannot be fetched", source)
+	}
+	return CachePathFor(f.CacheDir, source), nil
+}
+
+// isPath reports whether a source names a file on this machine.
+//
+// By its leading character, because that is the only honest signal available:
+// "Qwen/Qwen2.5-7B-Instruct" and "models/qwen.gguf" are indistinguishable as
+// strings, and an operator writing a local file writes an absolute path. The
+// cost of being wrong in this direction is that a relative path is passed to
+// the engine untouched, which is what happened before this code existed.
+func isPath(source string) bool {
+	return strings.HasPrefix(source, "/") ||
+		strings.HasPrefix(source, "./") ||
+		strings.HasPrefix(source, "../")
+}
+
+// isRemote reports whether a source has to be fetched at all.
+func isRemote(source string) bool {
+	return strings.HasPrefix(source, "hf:") ||
+		strings.HasPrefix(source, "http://") ||
+		strings.HasPrefix(source, "https://")
+}
+
+// resolveURL turns a source into something to GET.
+//
+// hf: names one file in a repository, which is what a GGUF is. A whole
+// repository of shards is deliberately not handled here: vLLM and SGLang
+// download those themselves and do it well, and reimplementing a snapshot
+// downloader to compete with them would be work spent to arrive back where we
+// started.
+func resolveURL(source string) (string, error) {
+	if rest, ok := strings.CutPrefix(source, "hf:"); ok {
+		parts := strings.Split(rest, "/")
+		if len(parts) < 3 {
+			return "", fmt.Errorf(
+				"%s names a repository, not a file; give the file as hf:org/repo/model.gguf, "+
+					"or use the repository id directly and let the engine download it", source)
+		}
+		repo := strings.Join(parts[:2], "/")
+		file := strings.Join(parts[2:], "/")
+		return "https://huggingface.co/" + repo + "/resolve/main/" + file, nil
+	}
+	if _, err := url.Parse(source); err != nil {
+		return "", fmt.Errorf("%s is not a usable source: %w", source, err)
+	}
+	return source, nil
+}

internal/agent/weights/fetch_test.go

@@ -0,0 +1,289 @@
+package weights
+
+import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func serveBytes(t *testing.T, body []byte, hits *atomic.Int64) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if hits != nil {
+			hits.Add(1)
+		}
+		http.ServeContent(w, r, "model.gguf", time.Time{}, strings.NewReader(string(body)))
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestALocalPathIsUsedNeverCopied(t *testing.T) {
+	// A path is what an operator writes when the file is already there —
+	// local disk, a Lustre mount, an image exported over the fabric. A
+	// downloader that second-guessed that would copy a hundred gigabytes that
+	// were already present.
+	dir := t.TempDir()
+	path := filepath.Join(dir, "model.gguf")
+	if err := os.WriteFile(path, []byte("weights"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	f := &Fetcher{CacheDir: t.TempDir()}
+
+	got, err := f.Ensure(t.Context(), path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != path {
+		t.Errorf("returned %q, want the original path", got)
+	}
+	if entries, _ := os.ReadDir(f.CacheDir); len(entries) != 0 {
+		t.Errorf("the cache was written to for a local file: %v", entries)
+	}
+}
+
+func TestAMissingLocalPathSaysWhatIsWrong(t *testing.T) {
+	// The commonest setup mistake, and "no such file" from an engine twenty
+	// minutes into a load is a poor way to learn it.
+	f := &Fetcher{CacheDir: t.TempDir()}
+	_, err := f.Ensure(t.Context(), filepath.Join(t.TempDir(), "absent.gguf"))
+	if !errors.Is(err, ErrNotLocal) {
+		t.Fatalf("err = %v, want ErrNotLocal", err)
+	}
+	if !strings.Contains(err.Error(), "absent.gguf") {
+		t.Errorf("the error does not name the file: %v", err)
+	}
+}
+
+func TestARemoteFileIsFetchedOnceAndReused(t *testing.T) {
+	var hits atomic.Int64
+	srv := serveBytes(t, []byte("the weights"), &hits)
+	f := &Fetcher{CacheDir: t.TempDir()}
+
+	first, err := f.Ensure(t.Context(), srv.URL+"/model.gguf")
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, err := os.ReadFile(first)
+	if err != nil || string(body) != "the weights" {
+		t.Fatalf("fetched content = %q, err %v", body, err)
+	}
+
+	second, err := f.Ensure(t.Context(), srv.URL+"/model.gguf")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if second != first {
+		t.Errorf("the second call returned a different path: %q vs %q", second, first)
+	}
+	if hits.Load() != 1 {
+		t.Errorf("the file was fetched %d times; the cache did not hold", hits.Load())
+	}
+}
+
+func TestConcurrentRequestsProduceOneDownload(t *testing.T) {
+	// Eight shards of a tensor-parallel replica start at the same instant and
+	// want the same file. Eight downloads is a self-inflicted denial of service
+	// on whatever is serving them.
+	var hits atomic.Int64
+	srv := serveBytes(t, []byte("shared weights"), &hits)
+	f := &Fetcher{CacheDir: t.TempDir()}
+
+	var wg sync.WaitGroup
+	paths := make([]string, 8)
+	errs := make([]error, 8)
+	for i := range 8 {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			paths[i], errs[i] = f.Ensure(t.Context(), srv.URL+"/model.gguf")
+		}()
+	}
+	wg.Wait()
+
+	for i, err := range errs {
+		if err != nil {
+			t.Fatalf("caller %d failed: %v", i, err)
+		}
+		if paths[i] != paths[0] {
+			t.Errorf("caller %d got %q, caller 0 got %q", i, paths[i], paths[0])
+		}
+	}
+	if n := hits.Load(); n != 1 {
+		t.Errorf("%d downloads for eight concurrent callers", n)
+	}
+}
+
+func TestAPartialDownloadIsNeverVisibleAsTheModel(t *testing.T) {
+	// A partial file under the real name would be found by the next scan,
+	// reported as present, and handed to an engine that fails on it minutes
+	// later — the exact failure all of this exists to prevent.
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Length", "1000")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("only the first bytes"))
+		// and then the connection dies without the rest
+		if fl, ok := w.(http.Flusher); ok {
+			fl.Flush()
+		}
+		panic(http.ErrAbortHandler)
+	}))
+	defer srv.Close()
+
+	cache := t.TempDir()
+	f := &Fetcher{CacheDir: cache}
+	if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf"); err == nil {
+		t.Fatal("a truncated download was reported as success")
+	}
+	entries, _ := os.ReadDir(cache)
+	for _, e := range entries {
+		if !strings.HasSuffix(e.Name(), ".part") {
+			t.Errorf("a non-part file was left behind: %q", e.Name())
+		}
+	}
+}
+
+func TestAnInterruptedDownloadResumes(t *testing.T) {
+	// A forty gigabyte fetch that fails at ninety percent must not start again
+	// from nothing.
+	full := []byte(strings.Repeat("x", 4096))
+	var ranges []string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ranges = append(ranges, r.Header.Get("Range"))
+		http.ServeContent(w, r, "model.gguf", time.Time{}, strings.NewReader(string(full)))
+	}))
+	defer srv.Close()
+
+	cache := t.TempDir()
+	f := &Fetcher{CacheDir: cache}
+	target, _ := f.cachePath(srv.URL + "/model.gguf")
+	// Half a file already on disk, as an interrupted attempt would leave.
+	if err := os.WriteFile(target+".part", full[:2048], 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	got, err := f.Ensure(t.Context(), srv.URL+"/model.gguf")
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, _ := os.ReadFile(got)
+	if len(body) != len(full) {
+		t.Errorf("resumed file is %d bytes, want %d", len(body), len(full))
+	}
+	if len(ranges) == 0 || !strings.Contains(ranges[0], "2048") {
+		t.Errorf("no range request was made: %v", ranges)
+	}
+}
+
+func TestAnHFSourceNamingARepositoryExplainsItself(t *testing.T) {
+	// A whole repository of shards is deliberately not handled here: vLLM and
+	// SGLang download those themselves and do it well. Saying so beats a
+	// confusing 404.
+	_, err := resolveURL("hf:Qwen/Qwen2.5-Coder-32B-Instruct")
+	if err == nil {
+		t.Fatal("a bare repository was accepted as a file")
+	}
+	for _, want := range []string{"model.gguf", "let the engine"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("the error does not suggest what to do (%q): %v", want, err)
+		}
+	}
+}
+
+func TestAnHFFileResolvesToItsDownloadURL(t *testing.T) {
+	got, err := resolveURL("hf:Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/qwen2.5-coder-3b-instruct-q4_k_m.gguf")
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "https://huggingface.co/Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/resolve/main/qwen2.5-coder-3b-instruct-q4_k_m.gguf"
+	if got != want {
+		t.Errorf("resolved to\n  %s\nwant\n  %s", got, want)
+	}
+}
+
+func TestTwoSourcesWithTheSameFilenameDoNotCollide(t *testing.T) {
+	f := &Fetcher{CacheDir: t.TempDir()}
+	a, _ := f.cachePath("https://one.example/model.gguf")
+	b, _ := f.cachePath("https://two.example/model.gguf")
+	if a == b {
+		t.Fatalf("both sources cache to %q", a)
+	}
+	// And a person looking at the directory can still tell what is in it.
+	for _, p := range []string{a, b} {
+		if !strings.HasSuffix(p, "model.gguf") {
+			t.Errorf("cache path %q does not keep the filename", p)
+		}
+	}
+}
+
+func TestProgressIsReportedWithAPercentage(t *testing.T) {
+	// A missing model is the difference between a replica that is slow to start
+	// and one that is stuck, and without this they look identical.
+	srv := serveBytes(t, []byte(strings.Repeat("y", 8192)), nil)
+	var seen []Progress
+	f := &Fetcher{
+		CacheDir:   t.TempDir(),
+		OnProgress: func(p Progress) { seen = append(seen, p) },
+	}
+	if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf"); err != nil {
+		t.Fatal(err)
+	}
+	if len(seen) == 0 {
+		t.Fatal("no progress was reported")
+	}
+	last := seen[len(seen)-1]
+	if last.Percent() != 100 {
+		t.Errorf("final progress is %d%%, want 100 (%+v)", last.Percent(), last)
+	}
+}
+
+func TestPercentIsUnknownRatherThanZeroWithoutATotal(t *testing.T) {
+	// A server that sends no Content-Length is common, and reporting 0% for
+	// a download that is nearly finished is worse than admitting ignorance.
+	if got := (Progress{BytesDone: 500}).Percent(); got != -1 {
+		t.Errorf("Percent with no total = %d, want -1", got)
+	}
+}
+
+func TestNoCacheDirectoryIsAClearRefusal(t *testing.T) {
+	f := &Fetcher{}
+	_, err := f.Ensure(t.Context(), "https://example.invalid/model.gguf")
+	if err == nil || !strings.Contains(err.Error(), "cache directory") {
+		t.Errorf("err = %v, want a complaint about the cache directory", err)
+	}
+}
+
+func TestABareModelIdIsLeftToTheEngine(t *testing.T) {
+	// "Qwen/Qwen2.5-7B-Instruct" is a documented source that vLLM and SGLang
+	// resolve themselves. Treating it as a missing file broke every existing
+	// vLLM profile, which is how this was found.
+	f := &Fetcher{CacheDir: t.TempDir()}
+	for _, id := range []string{
+		"Qwen/Qwen2.5-7B-Instruct",
+		"meta-llama/Llama-3.3-70B-Instruct",
+	} {
+		got, err := f.Ensure(t.Context(), id)
+		if err != nil {
+			t.Errorf("%s was refused: %v", id, err)
+		}
+		if got != id {
+			t.Errorf("%s was rewritten to %q", id, got)
+		}
+	}
+}
+
+func TestAnAbsolutePathThatIsMissingIsStillAnError(t *testing.T) {
+	// The other half: an operator writing a path means a file, and a missing
+	// one should be said plainly rather than handed to an engine.
+	f := &Fetcher{CacheDir: t.TempDir()}
+	if _, err := f.Ensure(t.Context(), "/models/definitely-not-here.gguf"); !errors.Is(err, ErrNotLocal) {
+		t.Errorf("err = %v, want ErrNotLocal", err)
+	}
+}

internal/agent/weights/gc.go

@@ -0,0 +1,99 @@
+package weights
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+// cacheNamePattern matches a filename this package created.
+//
+// The eight-byte hash of the source, hex encoded, then a dash, then the
+// original filename — exactly what cachePath writes. It is the only mark of
+// ownership there is, and it has to be exact: everything else in a model
+// directory was put there by the operator, and deleting one of those files
+// could cost a week of someone's bandwidth.
+var cacheNamePattern = regexp.MustCompile(`^[0-9a-f]{16}-`)
+
+// CachePathFor is where a source would be cached in the given directory.
+//
+// Exported so the control plane can work out which files in a node's reported
+// list are ones llmgw fetched for a model it still wants, without asking the
+// node anything. The agent and the control plane must agree on this name
+// exactly, which is why there is one function and not two.
+func CachePathFor(dir, source string) string {
+	sum := sha256.Sum256([]byte(source))
+	name := filepath.Base(source)
+	if name == "" || name == "." || name == "/" {
+		name = "weights"
+	}
+	return filepath.Join(dir, hex.EncodeToString(sum[:8])+"-"+name)
+}
+
+// Owned reports whether a path is one this package created.
+//
+// The dividing line of the whole feature. A file without the cache name was
+// put there by the operator — by scp, by a shared mount, by hand — and llmgw
+// has no business deleting it however unreferenced it looks. A file with the
+// cache name is one llmgw downloaded, and llmgw can take it back.
+func Owned(path string) bool {
+	return cacheNamePattern.MatchString(filepath.Base(path))
+}
+
+// Orphans lists cached files that no wanted source accounts for.
+//
+// `have` is what a node reports on disk and `wanted` is every source in the
+// catalog; both are passed in so this is a calculation rather than a scan, and
+// so it can be run on the control plane against a node's reported inventory.
+//
+// Only files this package created are ever returned. A source that is not
+// fetched — a plain path, or a model id an engine resolves itself — produces
+// no cache name and so protects nothing and deletes nothing, which is correct:
+// those files are not in the cache to begin with.
+func Orphans(have []string, wanted []string) []string {
+	keep := map[string]struct{}{}
+	for _, src := range wanted {
+		if !isRemote(src) {
+			// Nothing was fetched for it, so nothing in the cache belongs to
+			// it. Computing a name here would reserve a file that does not
+			// exist and could never be an orphan anyway.
+			continue
+		}
+		// By base name rather than full path: the control plane does not know
+		// each node's cache directory, and the name is what identifies the
+		// file wherever the node keeps it.
+		keep[filepath.Base(CachePathFor("", src))] = struct{}{}
+	}
+
+	var out []string
+	seen := map[string]struct{}{}
+	for _, path := range have {
+		base := filepath.Base(path)
+		if !Owned(base) {
+			continue
+		}
+		if _, wanted := keep[base]; wanted {
+			continue
+		}
+		if _, dup := seen[path]; dup {
+			continue
+		}
+		seen[path] = struct{}{}
+		out = append(out, path)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// PartialName reports whether a path is the temporary file a fetch writes to.
+//
+// Those are not orphans in the same sense — a fetch may be running right now —
+// so they are named separately rather than swept up with the rest. A `.part`
+// left behind by a killed agent is real garbage, but only its own node can
+// tell whether it is stale, and nothing here can.
+func PartialName(path string) bool {
+	return strings.HasSuffix(path, ".part")
+}