Compare commits
No files matched your search
@@ -38,3 +38,11 @@ jobs:
|
|||||||
|
|
||||||
- name: Build
|
- name: Build
|
||||||
run: go build ./...
|
run: go build ./...
|
||||||
|
|
||||||
|
# The NVML backend needs cgo and cannot run without a driver, but it must
|
||||||
|
# at least keep compiling: it is the only source of NVLink topology and a
|
||||||
|
# break would otherwise surface on a GPU node, far from here.
|
||||||
|
- name: Build the NVML backend
|
||||||
|
env:
|
||||||
|
CGO_ENABLED: "1"
|
||||||
|
run: go build -tags nvml ./...
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
# Attaches binaries to a release when a version tag is pushed.
|
||||||
|
#
|
||||||
|
# Without this a release is a git tag, and installing llmgw on a node means
|
||||||
|
# having a Go toolchain and the source. The onboarding script copies a binary
|
||||||
|
# from wherever it is run, which works and quietly makes whoever runs it the
|
||||||
|
# build server.
|
||||||
|
name: Release
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags: ["v*"]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
# The built-in token every job is given carries whatever the instance's
|
||||||
|
# default mode allows, and under the restricted default that is read-only
|
||||||
|
# on releases — so the build would succeed and the publish would be
|
||||||
|
# refused. `contents` covers code and releases together, which is what
|
||||||
|
# creating a release needs: `releases: write` alone was refused, because
|
||||||
|
# creating one can create its tag.
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
# Tags are what the version string is built from, and a shallow clone
|
||||||
|
# has none of them.
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- uses: actions/setup-go@v5
|
||||||
|
with:
|
||||||
|
go-version-file: go.mod
|
||||||
|
cache: true
|
||||||
|
|
||||||
|
- name: Build
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
version="${GITHUB_REF_NAME:-$(git describe --tags --always)}"
|
||||||
|
mkdir -p dist
|
||||||
|
|
||||||
|
# CGO_ENABLED=0 is the whole point: the binary has to run on a node
|
||||||
|
# whose libc is older than the builder's, and llmgw has no C
|
||||||
|
# dependency — modernc.org/sqlite is pure Go precisely so this stays
|
||||||
|
# true.
|
||||||
|
for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64; do
|
||||||
|
os="${target%/*}"; arch="${target#*/}"
|
||||||
|
name="llmgw-${version}-${os}-${arch}"
|
||||||
|
echo "building $name"
|
||||||
|
CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build \
|
||||||
|
-trimpath \
|
||||||
|
-ldflags "-s -w -X gitea.mynodes.xyz/ilker/llmgw/internal/version.Version=${version}" \
|
||||||
|
-o "dist/${name}" ./cmd/llmgw
|
||||||
|
(cd dist && sha256sum "${name}" >> checksums.txt)
|
||||||
|
done
|
||||||
|
|
||||||
|
# An NVML build for nodes with NVLink, which the portable binary
|
||||||
|
# cannot serve.
|
||||||
|
#
|
||||||
|
# NVML is the only source of NVLink topology, its bindings need cgo,
|
||||||
|
# and cgo is exactly what CGO_ENABLED=0 above exists to avoid. So the
|
||||||
|
# portable binary silently falls back to taking the lowest free GPU
|
||||||
|
# indices — right on an SXM box with NVSwitch, and wrong on a PCIe
|
||||||
|
# box where NVLink bridges pair specific cards: a tensor group
|
||||||
|
# straddling two bridges runs several times slower and nothing says
|
||||||
|
# so. Shipping only the portable build meant nobody could have that
|
||||||
|
# topology without building it themselves.
|
||||||
|
#
|
||||||
|
# Dynamically linked against the host's libc by necessity, so it is
|
||||||
|
# published beside the portable one rather than instead of it.
|
||||||
|
if command -v gcc >/dev/null; then
|
||||||
|
nvml_name="llmgw-${version}-linux-amd64-nvml"
|
||||||
|
CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build \
|
||||||
|
-tags nvml -trimpath \
|
||||||
|
-ldflags "-s -w -X gitea.mynodes.xyz/ilker/llmgw/internal/version.Version=${version}" \
|
||||||
|
-o "dist/${nvml_name}" ./cmd/llmgw
|
||||||
|
(cd dist && sha256sum "${nvml_name}" >> checksums.txt)
|
||||||
|
else
|
||||||
|
echo "no C toolchain; skipping the nvml build" >&2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Proof the artifact runs at all before it is published. It cannot
|
||||||
|
# test the cross-compiled ones, which is why the native build is
|
||||||
|
# checked rather than assumed.
|
||||||
|
./dist/llmgw-${version}-linux-amd64 --version
|
||||||
|
cat dist/checksums.txt
|
||||||
|
|
||||||
|
- name: Attach to the release
|
||||||
|
env:
|
||||||
|
TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
# Gitea gives every job a GITEA_TOKEN, so this is not the usual
|
||||||
|
# "you forgot to add a secret" — an empty one means the job was denied
|
||||||
|
# a token, which is an instance setting rather than a repository one.
|
||||||
|
if [ -z "${TOKEN:-}" ]; then
|
||||||
|
echo "The job was given no GITEA_TOKEN, so nothing can be uploaded."
|
||||||
|
echo "Check Settings -> Actions -> General; the binaries are in dist/."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
# workflow_dispatch can be fired from a branch, where GITHUB_REF_NAME
|
||||||
|
# is the branch name. Uploading then would create a release called
|
||||||
|
# "main". The binaries are still built and kept as a check that the
|
||||||
|
# release build works; only the publishing is a tag's business.
|
||||||
|
if [ "${GITHUB_REF_TYPE:-}" != "tag" ]; then
|
||||||
|
echo "not a tag (ref=${GITHUB_REF_NAME}); built, not published"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
tag="${GITHUB_REF_NAME}"
|
||||||
|
api="${GITHUB_SERVER_URL}/api/v1/repos/${GITHUB_REPOSITORY}"
|
||||||
|
|
||||||
|
# Gitea materialises a release row for every pushed tag, so this
|
||||||
|
# nearly always finds one and the create below is for a tag pushed
|
||||||
|
# some other way.
|
||||||
|
#
|
||||||
|
# The id is read with grep rather than sed because sed's .* is greedy:
|
||||||
|
# "id" appears again inside the nested author object, so the old
|
||||||
|
# pattern took the author's id when it matched at all — and it did not
|
||||||
|
# match "id": 13 with a space, which is what actually happened. The
|
||||||
|
# first "id" in a release object is the release's own.
|
||||||
|
release_id() {
|
||||||
|
curl -sS -H "Authorization: token $TOKEN" "$api/releases/tags/$tag" \
|
||||||
|
| grep -o '"id"[[:space:]]*:[[:space:]]*[0-9]*' \
|
||||||
|
| head -1 | grep -o '[0-9]*$'
|
||||||
|
}
|
||||||
|
|
||||||
|
id=$(release_id || true)
|
||||||
|
if [ -z "$id" ]; then
|
||||||
|
echo "no release for $tag yet; creating one"
|
||||||
|
out=$(curl -sS -w '\n%{http_code}' -X POST \
|
||||||
|
-H "Authorization: token $TOKEN" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d "{\"tag_name\":\"$tag\",\"name\":\"$tag\"}" \
|
||||||
|
"$api/releases")
|
||||||
|
echo "create returned $(printf '%s' "$out" | tail -1)"
|
||||||
|
# Re-read rather than parsing the create response: a 409 means it
|
||||||
|
# was there all along, which is a success for our purposes.
|
||||||
|
id=$(release_id || true)
|
||||||
|
[ -n "$id" ] || printf '%s\n' "$out" | head -5 >&2
|
||||||
|
fi
|
||||||
|
[ -n "$id" ] || { echo "could not find or create the release" >&2; exit 1; }
|
||||||
|
echo "release id $id"
|
||||||
|
|
||||||
|
for f in dist/*; do
|
||||||
|
echo "uploading $(basename "$f")"
|
||||||
|
curl -sS -f -X POST -H "Authorization: token $TOKEN" \
|
||||||
|
-F "attachment=@$f" \
|
||||||
|
"$api/releases/$id/assets?name=$(basename "$f")" >/dev/null \
|
||||||
|
|| { echo "upload of $(basename "$f") was refused" >&2; exit 1; }
|
||||||
|
done
|
||||||
|
echo "attached $(ls dist | wc -l) files to $tag"
|
||||||
+621
-4
@@ -7,15 +7,632 @@ is only released once its **exit criterion** is met, not when its code lands.
|
|||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| v0.1.0 | 0 | Skeleton and sim-engine | released |
|
| v0.1.0 | 0 | Skeleton and sim-engine | released |
|
||||||
| v0.2.0 | 1 | Static router: streaming proxy, filter/score/pick | released |
|
| v0.2.0 | 1 | Static router: streaming proxy, filter/score/pick | released |
|
||||||
| v0.3.0 | 1.5 | Admission control: priority bands, saturation detector | in progress |
|
| v0.3.0 | 1.5 | Admission control: priority bands, saturation detector | released |
|
||||||
| v0.4.0 | 2 | Agent: GPU/IB discovery, single-node engines | planned |
|
| v0.4.0 | 2 | Agent: GPU/IB discovery, single-node engines | released as part of v0.5.0 |
|
||||||
| v0.5.0 | 3 | Control plane: catalog, placement, reconciler | planned |
|
| v0.5.0 | 3 | Control plane: catalog, placement, reconciler | released |
|
||||||
| v0.6.0 | 4 | Multi-node replicas and fabric awareness | planned |
|
| v0.6.0 | 4 | Multi-node replicas and fabric awareness | released |
|
||||||
| v1.0.0 | 5 | Tenancy, prefix affinity, benchmarking | planned |
|
| v1.0.0 | 5 | Tenancy, prefix affinity, benchmarking | planned |
|
||||||
|
|
||||||
Until v1.0.0 the minor version is the breaking one: configuration formats and Go
|
Until v1.0.0 the minor version is the breaking one: configuration formats and Go
|
||||||
APIs change between minors without a deprecation period.
|
APIs change between minors without a deprecation period.
|
||||||
|
|
||||||
|
## Unreleased
|
||||||
|
|
||||||
|
### Memory notes for agents, and threads without API keys
|
||||||
|
|
||||||
|
Notes an agent writes for itself and reads back in later conversations, offered
|
||||||
|
as REST (`/v1/memory`) and as MCP tools at `/mcp` (`remember`, `recall`,
|
||||||
|
`forget`). A note is a key, text and tags, searched by words or tag, private to
|
||||||
|
a project and to the agent id in `x-llmgw-agent`. llmgw does not inject the
|
||||||
|
tools into chat requests: an agent that is not given the server never sees
|
||||||
|
memory.
|
||||||
|
|
||||||
|
Off for every project until someone runs `llmgw keys memory --retain-hours N`,
|
||||||
|
and the retention is required, as it is for threads.
|
||||||
|
|
||||||
|
On a gateway with no authentication, a `threads:` block (`store`,
|
||||||
|
`retain_hours`) now keeps conversations and notes in one shared pool. Before,
|
||||||
|
`x-llmgw-thread` was silently ignored there; now a gateway configured to keep
|
||||||
|
nothing refuses it with an explanation instead. The shared pool is for one
|
||||||
|
trusted machine: anyone who can reach the port and knows an id can read it.
|
||||||
|
|
||||||
|
### Conversations that can move between models
|
||||||
|
|
||||||
|
A thread a client names with `x-llmgw-thread`, carrying only its new turn.
|
||||||
|
llmgw replays the stored history to whichever model is serving, which is what
|
||||||
|
lets a conversation begun on a large model continue on a small one — including
|
||||||
|
when a spent budget moved it there and the caller never chose.
|
||||||
|
|
||||||
|
Off for every project until someone runs `llmgw keys threads`. Until then llmgw
|
||||||
|
stores no message content anywhere, as it never has, and that default is worth
|
||||||
|
more than the feature. The retention is required rather than optional, because
|
||||||
|
a conversation store with no expiry is a transcript of everything a team ever
|
||||||
|
typed kept forever by omission, and it is measured from the last turn so a
|
||||||
|
conversation in progress does not expire underneath it.
|
||||||
|
|
||||||
|
History that does not fit the serving model is trimmed oldest-first and the
|
||||||
|
reply says how many turns went, in `x-llmgw-thread-trimmed`. System messages
|
||||||
|
are never dropped — losing one silently changes what the model is for while it
|
||||||
|
carries on answering — and neither is the turn the caller just sent.
|
||||||
|
|
||||||
|
Threads are scoped to a project, and reading, writing or deleting one that
|
||||||
|
belongs to another answers exactly as one that never existed: two answers would
|
||||||
|
let a caller map which ids are real in projects they cannot read.
|
||||||
|
|
||||||
|
Capturing the reply is opt-in and bounded, so a request without a thread
|
||||||
|
accumulates nothing on the hottest path in the system, and one with a thread
|
||||||
|
stops at a cap rather than growing to it.
|
||||||
|
|
||||||
|
`DELETE /v1/threads/{id}` forgets a conversation, as a client rather than as an
|
||||||
|
operator: the person who said the thing should not have to ask someone to read
|
||||||
|
it before it can go.
|
||||||
|
|
||||||
|
### Disk, and taking back only what llmgw put there
|
||||||
|
|
||||||
|
Disk was the one resource nothing reported. `statfs` was already being called
|
||||||
|
on every model directory, but only to ask whether the filesystem was a network
|
||||||
|
one — the free space in the same struct was discarded, so the only way to find
|
||||||
|
out why a node was filling up was to ssh to it.
|
||||||
|
|
||||||
|
Nodes now report it, per filesystem rather than per configured directory: free
|
||||||
|
space belongs to the disk, and two model directories on one disk are one pool
|
||||||
|
of bytes. Shared filesystems are kept out of a node's own figure, because space
|
||||||
|
on a filer belongs to everyone mounting it and twenty nodes would otherwise
|
||||||
|
each report the same free terabyte. It shows in `ctl nodes`, the dashboard and
|
||||||
|
Prometheus, and a node that reported nothing reads as unknown everywhere — an
|
||||||
|
older agent must not look like a full disk.
|
||||||
|
|
||||||
|
A profile may declare `min_disk_gb`, and a node without the room is no longer a
|
||||||
|
candidate, with a reason `explain-placement` prints. Skipped on a node that
|
||||||
|
already holds the weights: it needs no room for them, and without that the node
|
||||||
|
best placed to run a model is the one most likely to be rejected.
|
||||||
|
|
||||||
|
`ctl weights` lists model files llmgw downloaded for models no longer in the
|
||||||
|
catalog, and `--collect` deletes them. The design is mostly about what it will
|
||||||
|
not do. Only files llmgw fetched are candidates, and that is a calculation
|
||||||
|
rather than a heuristic — the cache name is a hash of the source, so the wanted
|
||||||
|
set is computable exactly. Anything an operator put on a node is left alone
|
||||||
|
however unreferenced it looks. Each node checks every path again before
|
||||||
|
unlinking it, refusing anything outside its cache directory or without the
|
||||||
|
cache name, so a control plane that is wrong cannot talk a node into losing a
|
||||||
|
model that took a week to copy.
|
||||||
|
|
||||||
|
### Fixes
|
||||||
|
|
||||||
|
- **A capability was lost at an indirection.** The controller is handed a
|
||||||
|
forwarding dispatcher, and reaches past its interface by type assertion to
|
||||||
|
find what lies beyond scheduling. Deleting weights was implemented on the
|
||||||
|
agent service and not on the forwarder, so it was simply absent at runtime —
|
||||||
|
with the whole suite green, because the dispatchers the tests use do not
|
||||||
|
implement it either and its absence there is the expected case.
|
||||||
|
- **`ctl tool-check` blamed the engine for what the model did.** It had one
|
||||||
|
verdict for a failed attempt and told operators not to try another model.
|
||||||
|
Measured: the engine renders the tool prompt correctly, the coder models do
|
||||||
|
not use the `<tool_call>` wrapper it asks for, and `qwen2.5-3b-instruct` on
|
||||||
|
the same engine returns a populated `tool_calls` array. A failed attempt is
|
||||||
|
now two verdicts, told apart by whether the reply used the wrapper.
|
||||||
|
|
||||||
|
## v0.6.0 — 2026-09-18
|
||||||
|
|
||||||
|
Phase 4: a replica can be bigger than one machine, and the fabric it spans
|
||||||
|
decides where it is allowed to go. Around that, everything a cluster needs
|
||||||
|
before anyone other than its operator is let near it — keys, budgets, a way in
|
||||||
|
through a browser, and enough visibility to answer "why is nothing scheduling".
|
||||||
|
|
||||||
|
### Multi-node replicas
|
||||||
|
|
||||||
|
A replica is now a set of shards with ranks, a leader and a bootstrap address,
|
||||||
|
rather than one process. Two invariants are enforced rather than documented: a
|
||||||
|
replica never spans two fabrics, and a tensor-parallel group never mixes GPU
|
||||||
|
classes. Both are the kind of mistake that does not fail — it hangs, or it runs
|
||||||
|
several times slower, and nothing says which.
|
||||||
|
|
||||||
|
A replica missing a shard is not ready. It was previously derived from the
|
||||||
|
shards that had reported, so a two-node replica with one node up looked
|
||||||
|
servable and took traffic into a model that could not answer.
|
||||||
|
|
||||||
|
Adoption after a restart happens one agent at a time, because that is how
|
||||||
|
agents reconnect. Merging them as they arrive is what keeps a multi-node
|
||||||
|
replica whole instead of being torn into fragments that are then killed as
|
||||||
|
strays — and an adopted engine is health-checked before it is called ready,
|
||||||
|
rather than being assumed alive because its process exists.
|
||||||
|
|
||||||
|
### Fabrics are distribution domains too
|
||||||
|
|
||||||
|
An isolated fabric is not only a compute boundary: weights present in one are
|
||||||
|
not present in the other. Node weight inventories are reported and summarised
|
||||||
|
per fabric, so "the model is on the cluster" is never mistaken for "the model
|
||||||
|
is reachable from the nodes that would run it". A fabric name that matches
|
||||||
|
nothing a node actually has is refused at registration rather than becoming an
|
||||||
|
unschedulable replica hours later.
|
||||||
|
|
||||||
|
### Who may use it
|
||||||
|
|
||||||
|
API keys, tenants and per-model allowlists, on SQLite (`modernc.org/sqlite`, so
|
||||||
|
still no cgo). Rate limits and budgets are enforced on the request path; usage
|
||||||
|
is written off it, in batches, so accounting never adds latency to inference.
|
||||||
|
Budgets survive a restart — counters are rebuilt from the store, because a
|
||||||
|
limit anyone can clear by bouncing the process is not a limit.
|
||||||
|
|
||||||
|
A spent budget is now served a cheaper model where the catalog offers one,
|
||||||
|
rather than refused. For a team trying models out, being dropped to something
|
||||||
|
smaller is a far better answer than a wall, and the response says which model
|
||||||
|
answered.
|
||||||
|
|
||||||
|
People can sign up, be approved, and issue and revoke their own keys. A key is
|
||||||
|
shown once; only its hash is kept.
|
||||||
|
|
||||||
|
### The browser surface
|
||||||
|
|
||||||
|
A landing page that says how to connect, a read-only cluster dashboard, and an
|
||||||
|
in-browser chat that never puts an API key in the browser. The dashboard shows
|
||||||
|
who is using the cluster and what they are running — per person and per model,
|
||||||
|
with requests, tokens, errors, spend and latency.
|
||||||
|
|
||||||
|
All of it is served from the client-facing port behind a session, deliberately
|
||||||
|
not from the admin port, which also serves pprof — and pprof can dump the heap
|
||||||
|
that holds every API key in plaintext.
|
||||||
|
|
||||||
|
### Models without a restart
|
||||||
|
|
||||||
|
The catalog is live. Models can be added and removed from the command line or
|
||||||
|
the API, take effect immediately, and outlive a restart; configuration still
|
||||||
|
wins over the store, and the whole catalog is validated before anything is
|
||||||
|
written.
|
||||||
|
|
||||||
|
A model is only listed once it can actually serve. It was listed as soon as it
|
||||||
|
was known, so the first thing a new user did was ask for a model that returned
|
||||||
|
503.
|
||||||
|
|
||||||
|
### Loading on demand, and letting go
|
||||||
|
|
||||||
|
A model whose scaling allows it is started when someone asks for it, and
|
||||||
|
stopped when nobody has for a while. An idle replica is evicted to make room
|
||||||
|
for one that was asked for, with guards that exist because each is a way to
|
||||||
|
lose work quietly: only on-demand models, only single-node replicas, never one
|
||||||
|
that has never been used, and never one younger than its minimum lifetime.
|
||||||
|
|
||||||
|
Unknown is not idle. A replica with no recorded use is left alone rather than
|
||||||
|
treated as idle since the epoch.
|
||||||
|
|
||||||
|
### Knowing whether it will run before it runs
|
||||||
|
|
||||||
|
- `ctl plan` works out whether a model fits the cluster and how it would be
|
||||||
|
split, in the precision the chosen engine can actually serve — recommending
|
||||||
|
a quantisation the engine cannot load would cause the failure the command
|
||||||
|
exists to prevent.
|
||||||
|
- `ctl tool-check` verifies llmgw's engine flags against the engines actually
|
||||||
|
installed. Flags drift between releases, and the alternative is finding out
|
||||||
|
during a model load.
|
||||||
|
- Placement now knows a model's context length and memory footprint, and node
|
||||||
|
RAM, so a CPU profile is not placed on a machine that cannot hold it.
|
||||||
|
- Placement waits for the fleet to settle after a restart rather than deciding
|
||||||
|
against a half-reported cluster.
|
||||||
|
|
||||||
|
### Weights
|
||||||
|
|
||||||
|
Nodes report the model files they can see, without llmgw owning the storage:
|
||||||
|
a shared filer, an image exported over the fabric, rsync or a copy made by hand
|
||||||
|
all look the same to placement, which only cares whether the files are there.
|
||||||
|
Reporting nothing means "not asked to look", not "has nothing".
|
||||||
|
|
||||||
|
Where the simple arrangement is not enough, a node can fetch a model by URL —
|
||||||
|
single-flight, resumable, written to a temporary name and renamed atomically,
|
||||||
|
so a killed download never leaves something that looks like weights.
|
||||||
|
|
||||||
|
### Seeing the fleet
|
||||||
|
|
||||||
|
Prometheus metrics for nodes, GPUs and replicas, collected from live state at
|
||||||
|
scrape time rather than from a background goroutine, and served on their own
|
||||||
|
endpoint rather than through the one that also exposes pprof. The same view is
|
||||||
|
in the dashboard, so nothing else needs installing to see it.
|
||||||
|
|
||||||
|
### Onboarding a node
|
||||||
|
|
||||||
|
Adding a node is one command. There are systemd units, a private CA and a
|
||||||
|
from-nothing install guide that starts with a machine that has no CUDA and no
|
||||||
|
binary.
|
||||||
|
|
||||||
|
### llama.cpp, and CPU profiles
|
||||||
|
|
||||||
|
A third engine adapter, and the reason for it: llama.cpp serves the same OpenAI
|
||||||
|
API without a GPU, so placement, supervision, scraping, scheduling and admission
|
||||||
|
can all be exercised on ordinary machines.
|
||||||
|
|
||||||
|
A profile may now omit `gpu_class`, which is how it says it runs on the CPU. Only
|
||||||
|
engines that can actually serve without a card may omit it — for vLLM an omitted
|
||||||
|
class stays an error, because otherwise a typo becomes a replica placed on a
|
||||||
|
machine with no hardware and diagnosed minutes later during model loading. CPU
|
||||||
|
replicas are spread across nodes rather than bin packed: unlike GPUs there is no
|
||||||
|
allocation to stop two of them sharing a machine's cores.
|
||||||
|
|
||||||
|
### Metric mappings are per engine
|
||||||
|
|
||||||
|
`Running`, `Waiting` and `KVCacheUsage` now have a vocabulary per engine rather
|
||||||
|
than one for the cluster. A mixed fleet has two names for the same three numbers,
|
||||||
|
and a single global mapping left half of it unmeasurable.
|
||||||
|
|
||||||
|
A mapping that leaves a metric unnamed now means the engine does not publish it,
|
||||||
|
and the field stays zero. Naming one the engine does not serve is still a contract
|
||||||
|
violation. `Running` and `Waiting` can never be omitted.
|
||||||
|
|
||||||
|
### Mutual TLS on the agent stream
|
||||||
|
|
||||||
|
A private CA (`llmgw ctl cert init-ca | issue-server | issue-client | inspect`),
|
||||||
|
certificates on both ends, and the control plane refusing any agent whose claimed
|
||||||
|
node id does not match its certificate's common name. Public authorities cannot
|
||||||
|
supply these: they stopped issuing the `clientAuth` usage.
|
||||||
|
|
||||||
|
Plaintext HTTP/2 remains the default. `agent_tls:` on the server and `tls:` on the
|
||||||
|
agent turn it on.
|
||||||
|
|
||||||
|
### Fixes
|
||||||
|
|
||||||
|
- **An agent told to stop did not stop.** Cancelling the context is not enough —
|
||||||
|
HTTP/2 watches it while the request body is being written, and a bidirectional
|
||||||
|
stream spends its life blocked on the read side. Measured on a deployed node:
|
||||||
|
`systemctl stop` took 90 seconds and ended in SIGKILL. Now 0.01 seconds.
|
||||||
|
- **An agent could not explain why it was refused.** The session cancelled its own
|
||||||
|
context and then asked that context whether it was shutting down, so every
|
||||||
|
stream error read as a clean close: no message, and the backoff reset.
|
||||||
|
- **A broken metric contract was invisible.** It was logged per scrape at debug
|
||||||
|
level, so a cluster could look healthy while every routing decision was made
|
||||||
|
blind. It is now a warning, once, with a matching line when it clears.
|
||||||
|
- **Token estimates came from the routing prefix, not the prompt.** Every prompt
|
||||||
|
over about 2 KB was undercounted, so tokens-per-minute limits let through far
|
||||||
|
more than they were set to.
|
||||||
|
- **Model-fitting double-counted overhead.** Whole-model overhead was charged per
|
||||||
|
rank and a 10% reserve taken twice, so `plan` reported that a 70B model in
|
||||||
|
bfloat16 does not fit two 80 GB cards. It does.
|
||||||
|
- **`x-llmgw-served-model` was never sent.** It was set after the proxy had
|
||||||
|
already written the response header, which is the one point at which headers
|
||||||
|
stop being changeable. A client had no way to tell a fallback from the model
|
||||||
|
it asked for.
|
||||||
|
- **Fleet metrics published nothing.** They were registered after the registry
|
||||||
|
had been built, so the collector existed and was never scraped.
|
||||||
|
- **GPU discovery broke on older drivers.** A field added for utilisation was
|
||||||
|
required rather than optional, so a driver that does not report it produced no
|
||||||
|
inventory at all instead of an inventory without that number.
|
||||||
|
- **Replica ids and ports were reused** while the processes holding them were
|
||||||
|
still alive.
|
||||||
|
|
||||||
|
## v0.5.0 — 2026-09-16
|
||||||
|
|
||||||
|
Phases 2 and 3. The cluster runs itself: agents report their hardware, the
|
||||||
|
control plane places replicas, and `ctl` makes all of it visible.
|
||||||
|
|
||||||
|
This release covers both phases because their exit criteria depended on the same
|
||||||
|
piece — the agent stream — and neither was demonstrable without it.
|
||||||
|
|
||||||
|
### ctl
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw ctl nodes | models | replicas | explain-placement | cordon | uncordon | drain
|
||||||
|
```
|
||||||
|
|
||||||
|
- `nodes` surfaces unhealthy GPUs and why they are excluded, which is usually
|
||||||
|
the answer when a node looks idle but nothing schedules onto it.
|
||||||
|
- `explain-placement` prints every candidate considered and the reason it was
|
||||||
|
rejected. "Unschedulable" on its own says nothing about which of six
|
||||||
|
constraints bit.
|
||||||
|
- `drain` is a rolling move, not a stop: the replica is drained, a replacement
|
||||||
|
is placed elsewhere, and the original is stopped only once the replacement is
|
||||||
|
serving. Bounded by a grace period, because draining the only node that can
|
||||||
|
host a model would otherwise never finish and leave it undrainable.
|
||||||
|
- `cordon` is the weaker operation: no new placement, nothing disturbed.
|
||||||
|
|
||||||
|
The management API is on the admin listener, not the client-facing one. The two
|
||||||
|
have different audiences: one serves inference to whoever holds an API key, the
|
||||||
|
other lets a caller cordon nodes.
|
||||||
|
|
||||||
|
`ctl logs` is not here. It needs a request/response pair the agent protocol does
|
||||||
|
not have. The failure path already ships an engine log tail with every status
|
||||||
|
report, so the case that matters most is covered; reading a healthy shard's
|
||||||
|
output on demand is not.
|
||||||
|
|
||||||
|
### Phases 2 and 3, in detail
|
||||||
|
|
||||||
|
Written as the work landed, which is why it reads as a log rather than as
|
||||||
|
notes. Everything here shipped in v0.5.0.
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- **IB discovery** (`internal/agent/discovery`): reads `/sys/class/infiniband`,
|
||||||
|
parses the formats sysfs actually uses (`"4: ACTIVE"`, `"200 Gb/sec (4X HDR)"`,
|
||||||
|
hex or decimal LIDs), and distinguishes native InfiniBand from RoCE. A node
|
||||||
|
without IB is not an error — it is a single-node island.
|
||||||
|
- **GPU discovery** via `nvidia-smi`, the cgo-free path. NVML needs cgo, which
|
||||||
|
would force the whole binary, server included, to be built with a C
|
||||||
|
toolchain. Health flags anything that should keep a card out of placement:
|
||||||
|
uncorrected ECC errors, and memory already held by another process.
|
||||||
|
- **GPU class derivation.** Sizes come from the driver in GiB, which does not
|
||||||
|
always match the marketing name: an L40S is sold as 48GB and reports 44.99
|
||||||
|
GiB, so its class is `l40s-45g`. Consistency matters more than matching the
|
||||||
|
box, because the number decides whether a model fits.
|
||||||
|
- **Engine adapters** for vLLM and SGLang: command and environment
|
||||||
|
construction, kept free of process handling so it is testable without GPUs.
|
||||||
|
vLLM supports both Ray and multiprocessing launch modes, because which one
|
||||||
|
works depends on the version.
|
||||||
|
- **NCCL environment generation**, the part plan §5.3 marks critical.
|
||||||
|
`NCCL_IB_HCA` lists only the chosen fabric's adapters — leaving it unset lets
|
||||||
|
NCCL pick an adapter on another fabric where the peer is unreachable, and the
|
||||||
|
job hangs rather than failing. Bootstrap is pinned to the management
|
||||||
|
interface for the same reason.
|
||||||
|
- **Validation before launch**: a tensor-parallel group may not mix GPU types,
|
||||||
|
the node/GPU/TP arithmetic has to add up, and an FP8 profile is refused below
|
||||||
|
compute capability 8.9 rather than silently falling back to a slower path.
|
||||||
|
- **Supervisor** (`internal/agent/supervisor`): processes run in their own
|
||||||
|
process group and are stopped as a group, because engines fork workers that
|
||||||
|
hold GPU memory and outlive a signal sent only to the leader. Bounded log
|
||||||
|
tail, a state machine that separates Starting from Loading (a 70B model takes
|
||||||
|
minutes, and an operator needs to know that is normal), and orphan adoption
|
||||||
|
through pid files fingerprinted with the command line — pids get reused, and
|
||||||
|
adopting an unrelated process is worse than leaking the original.
|
||||||
|
- **Readiness** is two questions: `/health` says the process is up, `/v1/models`
|
||||||
|
says the weights loaded. Routing on `/health` alone sends traffic into a model
|
||||||
|
that is still loading.
|
||||||
|
- **`llmgw agent inventory`** reports what the scheduler would see on a node,
|
||||||
|
as a table or JSON. Run it before adding a node: it is the fastest way to find
|
||||||
|
a card another process is using, or a port with no subnet manager.
|
||||||
|
|
||||||
|
- **NVML backend** behind the `nvml` build tag, which adds NVLink topology.
|
||||||
|
Nothing else exposes which GPUs share an NVLink domain, and that is what keeps
|
||||||
|
a tensor-parallel group off the slow path inside a node. `Discover` falls back
|
||||||
|
to nvidia-smi if NVML fails, and says so loudly rather than silently losing the
|
||||||
|
topology. CI compiles the tagged build so it cannot rot unnoticed.
|
||||||
|
|
||||||
|
### Not verified
|
||||||
|
|
||||||
|
The NVML path compiles but has never run against a driver — there is no GPU
|
||||||
|
machine to try it on. Treat the NVLink domains it reports as unverified until
|
||||||
|
someone runs `llmgw agent inventory` on a real node.
|
||||||
|
|
||||||
|
- **Control plane wiring** (`internal/control`): Ready replicas become router
|
||||||
|
endpoints, with no static configuration involved. The router keeps live state
|
||||||
|
for endpoints that survive an update — inflight counts, breaker state, drain
|
||||||
|
rate — so a replica that was already serving does not start from cold because
|
||||||
|
another one moved. The sink is an interface, so when the router and control
|
||||||
|
plane split into separate processes this becomes a watch stream and nothing
|
||||||
|
above it changes.
|
||||||
|
|
||||||
|
### Phase 3 exit criterion, measured in `test/e2e`
|
||||||
|
|
||||||
|
A node hosting a live replica is killed while four clients stream continuously.
|
||||||
|
The control plane marks it unreachable, then lost, rebuilds the replica on a
|
||||||
|
survivor, and republishes. Across three runs: **~370 requests completed, zero
|
||||||
|
failures**, and no replica remained on the dead node.
|
||||||
|
|
||||||
|
This proves the control logic, not the transport: the agent fleet is simulated,
|
||||||
|
and the real path still needs the stream.
|
||||||
|
|
||||||
|
### A real bug the race detector caught
|
||||||
|
|
||||||
|
`startReplica` read the replica's fields after publishing it to the map, and
|
||||||
|
`scaleUp` walked replica pointers outside the lock. Both were racing with agent
|
||||||
|
status reports, which arrive from another goroutine and mutate exactly those
|
||||||
|
fields. Under the simulated fleet the reports came back fast enough to hit the
|
||||||
|
window every time. Everything is snapshotted under the lock now and the
|
||||||
|
functions work from values.
|
||||||
|
|
||||||
|
### The agent stream
|
||||||
|
|
||||||
|
- **Protocol** (`api/proto/llmgw/v1`): one bidirectional connect-go stream
|
||||||
|
carrying registration, heartbeats and shard status upward, and start, stop and
|
||||||
|
drain commands downward. The agent dials out, because GPU nodes sit behind
|
||||||
|
firewalls far more often than a control plane does and an outbound connection
|
||||||
|
needs no inbound rule on every node in the fleet.
|
||||||
|
- **Registration carries the whole inventory**, so there is no separate
|
||||||
|
discovery call and a reconnecting agent refreshes what the scheduler knows.
|
||||||
|
The fabric labels travel with it — they cannot be inferred from the wire, so
|
||||||
|
losing them in transit would silently disable multi-node placement.
|
||||||
|
- **Agent service** (`internal/control/agentsvc`) is both the stream handler and
|
||||||
|
the reconciler's dispatcher: it holds the open streams, so it is the only
|
||||||
|
thing that can deliver a command to a node.
|
||||||
|
- **Agent** (`internal/agent`) validates a profile before launching anything —
|
||||||
|
an engine that refuses a profile does so after minutes of loading — then
|
||||||
|
supervises the process and reports every transition, including the engine's
|
||||||
|
output tail on failure.
|
||||||
|
- **Reconnects forever with jittered backoff.** A control plane restart must not
|
||||||
|
require touching every GPU node, and a fleet must not reconnect in lockstep
|
||||||
|
and knock the control plane over again. Engines keep serving throughout: the
|
||||||
|
router talks to them directly.
|
||||||
|
- **`llmgw agent` is real**, and `llmgw server` runs the control plane whenever
|
||||||
|
any model declares a profile. The static endpoint list remains for
|
||||||
|
router-only runs against sim-engines.
|
||||||
|
|
||||||
|
### Verified with real binaries
|
||||||
|
|
||||||
|
Server and agent as separate processes over a real stream: the agent registers,
|
||||||
|
a replica is placed, an engine is launched through the vLLM adapter's actual
|
||||||
|
argument construction, streaming traffic is served through the router, and
|
||||||
|
killing the agent moves the node through unreachable to lost with placement
|
||||||
|
explaining exactly why it can no longer run anything.
|
||||||
|
|
||||||
|
### Surviving a control plane restart
|
||||||
|
|
||||||
|
Not persisting replica state left one hole that had to be closed before it
|
||||||
|
mattered: restarting the control plane orphaned live engines. They kept holding
|
||||||
|
GPUs the inventory believed were free, so the next placement failed to allocate
|
||||||
|
with an error naming the new process rather than the old one.
|
||||||
|
|
||||||
|
The fix is on the agent, not in a database, and it cannot disagree with reality
|
||||||
|
the way a stale row can:
|
||||||
|
|
||||||
|
- The supervisor now writes the full shard spec, not just a pid and command
|
||||||
|
line. A pid identifies a process; the spec is what lets it be described back
|
||||||
|
to the control plane and matched to a replica.
|
||||||
|
- `Adopt` takes over a still-running engine from a previous agent. It cannot be
|
||||||
|
waited on — it is not this process's child — so it is watched by polling, and
|
||||||
|
stopped by signalling its group directly. Its earlier output is gone, and the
|
||||||
|
log says so rather than presenting emptiness as silence.
|
||||||
|
- Every `Register`, not only the first, reports what the agent is running. A
|
||||||
|
reconnect may be a control plane restart and has to recover the same way.
|
||||||
|
- The reconciler rebuilds replicas from those reports and reclaims their GPUs.
|
||||||
|
Anything whose model or profile left the catalog is stopped: nothing will
|
||||||
|
route to it and it is holding cards.
|
||||||
|
- An engine the agent cannot identify — a corrupt state file — is killed. A
|
||||||
|
process holding GPUs that nothing can describe is worse than no process,
|
||||||
|
because placement will hand those GPUs out again.
|
||||||
|
|
||||||
|
One mechanism covers agent restarts and control plane restarts.
|
||||||
|
|
||||||
|
### A test that was passing for no reason
|
||||||
|
|
||||||
|
`TestAgentReconnects` used `httptest.CloseClientConnections`, which does nothing
|
||||||
|
to an h2c stream: prior-knowledge h2c hijacks the connection, and httptest stops
|
||||||
|
tracking hijacked connections. The agent never reconnected and the assertion —
|
||||||
|
on the connection map, which still held a stale entry — passed anyway. Tests now
|
||||||
|
sever connections through their own listener and count opened streams.
|
||||||
|
|
||||||
|
### Multi-node launch
|
||||||
|
|
||||||
|
Placement and environment generation had landed early, which made multi-node
|
||||||
|
look further along than it was: a multi-node replica could not come up at all.
|
||||||
|
Three things were missing, and all three came down to two assumptions that do
|
||||||
|
not hold once a replica spans nodes.
|
||||||
|
|
||||||
|
**Not every rank serves HTTP.** The agent health-checked every shard at its own
|
||||||
|
port, but only the leader answers. Workers timed out, reported Failed, and took
|
||||||
|
the replica with them. `Engine.ServesHTTP(spec)` now says which ranks serve; a
|
||||||
|
worker is judged by staying alive instead. That is weaker evidence on purpose —
|
||||||
|
the replica's real readiness is the leader answering, which only happens once
|
||||||
|
every worker has joined.
|
||||||
|
|
||||||
|
**A shard is not always one process.** `Engine.Commands(spec)` returns a
|
||||||
|
sequence: every command but the last is a sidecar kept alive for the shard's
|
||||||
|
lifetime and torn down with it, and the last one's exit is the shard's exit.
|
||||||
|
vLLM under Ray is why — the leader runs a Ray head and then the server against
|
||||||
|
it, and a worker runs only the join, with no vLLM process at all. `RayCommand`
|
||||||
|
had been written and tested but never called, so `launch_mode: ray` would have
|
||||||
|
run `vllm serve` against a cluster that did not exist.
|
||||||
|
|
||||||
|
A command can also require the rendezvous point to be listening before the next
|
||||||
|
one starts, bounded by a timeout so a head that never appears fails the shard
|
||||||
|
rather than hanging it. Sidecars are killed with the shard: a Ray head left
|
||||||
|
behind holds the port the next replica needs.
|
||||||
|
|
||||||
|
**Start order was backwards.** Workers were started first, on the theory that
|
||||||
|
the head should wait for them. Both engines put the rendezvous point on rank 0,
|
||||||
|
so a worker started first has nothing to connect to and spends its startup
|
||||||
|
budget retrying. vLLM's server blocks until the Ray cluster has the GPUs it
|
||||||
|
needs, so the leader can start immediately and waits on its own.
|
||||||
|
|
||||||
|
None of this is verified against real engines. It is tested with stub processes;
|
||||||
|
the Ray sequencing and which launch mode a given vLLM version wants can only be
|
||||||
|
confirmed on hardware.
|
||||||
|
|
||||||
|
### Still missing for phase 2
|
||||||
|
|
||||||
|
- mTLS. The stream is plaintext HTTP/2 today, so it must stay on a trusted
|
||||||
|
network.
|
||||||
|
|
||||||
|
### Phase 3: placement
|
||||||
|
|
||||||
|
- **Cluster inventory** (`internal/control/inventory`): nodes, their hardware,
|
||||||
|
their fabric attachments, and what is allocated where. A re-registering agent
|
||||||
|
cannot clear an operator's cordon, and a heartbeat revives an unreachable node
|
||||||
|
without doing so either. A GPU with an ECC fault or another process on it is
|
||||||
|
not counted as free. Replicas are released whole, because they fail whole.
|
||||||
|
- **Placement** (`internal/control/placement`): picks the highest-priority
|
||||||
|
profile that fits and explains every candidate it rejected. Two constraints
|
||||||
|
are enforced structurally rather than checked afterwards — there is no code
|
||||||
|
path that can combine nodes from two fabrics, and only GPUs of the profile's
|
||||||
|
class are ever considered.
|
||||||
|
- **Bin packing** fills partly used nodes first, so whole nodes stay free for
|
||||||
|
large models that cannot be split. Multi-node replicas get the opposite
|
||||||
|
treatment: they are the least movable thing in the cluster, so they take the
|
||||||
|
emptiest machines.
|
||||||
|
- **NVLink-aware GPU selection** keeps a tensor-parallel group inside one NVLink
|
||||||
|
domain where the topology is known. Without the NVML build it falls back to
|
||||||
|
the lowest free indices, which is usually right but only by accident of how
|
||||||
|
vendors number slots.
|
||||||
|
- **Property-based invariant tests** (plan §13): 400 randomly generated
|
||||||
|
clusters, checking on every placement that a multi-node replica lives on one
|
||||||
|
fabric, that each shard receives only that fabric's adapters, and that no
|
||||||
|
placement mixes GPU classes or uses an unhealthy or already-allocated GPU.
|
||||||
|
The table-driven tests cover the cases someone thought of; this covers the
|
||||||
|
ones nobody did.
|
||||||
|
- **Catalog** (`internal/control/catalog`): models, profiles, aliases,
|
||||||
|
fallbacks and scaling bounds. Aliases and fallbacks are resolved at load
|
||||||
|
rather than at request time — a typo in a fallback should stop a deployment,
|
||||||
|
not surface during the outage when the fallback is finally needed. The
|
||||||
|
node/GPU/TP arithmetic is checked here too, so a bad profile fails at config
|
||||||
|
load instead of minutes into model loading.
|
||||||
|
- **Reconciler** (`internal/control/reconciler`): the desired-versus-actual
|
||||||
|
loop. It places one replica per pass, because placement depends on what is
|
||||||
|
already allocated and starting several from one snapshot would hand them the
|
||||||
|
same GPUs. Workers start before rank 0, since SGLang's ranks rendezvous and
|
||||||
|
vLLM under Ray needs its workers attached before the head serves.
|
||||||
|
- **Replica failure is whole-replica failure.** One dead shard makes the others
|
||||||
|
useless — they wait on collectives that will never complete while holding
|
||||||
|
GPUs — so the reconciler stops every shard and rebuilds, with a jittered
|
||||||
|
exponential backoff carried on the model rather than the replica it just
|
||||||
|
destroyed.
|
||||||
|
- **Node expiry in two stages.** Unreachable first, Lost later. A brief network
|
||||||
|
blip must not tear down replicas that are still serving, and rebuilding a
|
||||||
|
multi-node replica costs tens of minutes of model loading.
|
||||||
|
- **Scale-down drains before stopping**, so in-flight requests finish instead of
|
||||||
|
being cut off mid-stream.
|
||||||
|
|
||||||
|
## v0.3.0 — 2026-09-16
|
||||||
|
|
||||||
|
Flow control. A batch workload no longer starves interactive traffic.
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- **Admission control** (`internal/router/admission`), inserted between quota
|
||||||
|
checks and scheduling. It answers *when* a request may proceed; the scheduler
|
||||||
|
still answers *where* it goes. Keeping them apart is the point: an engine
|
||||||
|
queue is FIFO and knows nothing about tenants, so once a batch job is in it,
|
||||||
|
no amount of endpoint selection rescues the request behind it.
|
||||||
|
- **Saturation detector.** One number per pool, the worse of queue depth and KV
|
||||||
|
occupancy. Either signal alone misleads — a pool can have an empty queue and a
|
||||||
|
nearly full cache, or the reverse.
|
||||||
|
- **Priority bands.** Each band has a saturation ceiling; above it, that band
|
||||||
|
waits. The cluster then degrades from the bottom up. A ceiling of 1.0 means
|
||||||
|
"flow up to the healthy limit"; above 1 means never wait.
|
||||||
|
- **Tenant fairness.** Within a band, dispatch turns rotate between tenants, so
|
||||||
|
one caller sending a thousand requests occupies one slot in the rotation
|
||||||
|
rather than a thousand.
|
||||||
|
- **Bounded queues.** Capacity per band (immediate 503 when full) and a TTL
|
||||||
|
(504), because sending the GPU work the client has already abandoned helps
|
||||||
|
nobody. Cancelled requests leave the queue instead of holding capacity.
|
||||||
|
- **Metrics:** `llmgw_admission_wait_seconds`, `llmgw_admission_rejected_total`,
|
||||||
|
`llmgw_admission_queue_depth`, `llmgw_pool_saturation`. Responses carry
|
||||||
|
`x-llmgw-queue-ms` when a request waited.
|
||||||
|
|
||||||
|
### Phase 1.5 exit criterion, measured in `test/e2e`
|
||||||
|
|
||||||
|
While 500 concurrent bronze requests run continuously, interactive p99 in the
|
||||||
|
platinum band stays at **1.04–1.24x** its idle value, against a 1.5x budget.
|
||||||
|
With admission switched off the same load pushes it to **55–60x**. Both
|
||||||
|
directions are asserted, so the test fails if admission regresses *and* if the
|
||||||
|
load generator ever stops being a real load.
|
||||||
|
|
||||||
|
### Two lags, two different fixes
|
||||||
|
|
||||||
|
Getting this right took longer than the queueing itself:
|
||||||
|
|
||||||
|
- Engine metrics are scraped, so they lag by an interval. The router's inflight
|
||||||
|
count does not — but it includes *running* requests, and treating those as
|
||||||
|
queue depth makes a healthy engine look overloaded every interval. What is
|
||||||
|
genuinely queued is `inflight − batch capacity`, and batch capacity is not in
|
||||||
|
the metric contract, so it is learned as the high-water mark of observed
|
||||||
|
`running`.
|
||||||
|
- Between admission granting a request and the router dispatching it, the
|
||||||
|
request is invisible to every signal. A simultaneous burst therefore passed
|
||||||
|
the check en masse — exactly the batch arrival pattern this layer exists to
|
||||||
|
survive. The controller now counts its own outstanding leases against
|
||||||
|
dispatched inflight.
|
||||||
|
|
||||||
|
### Known limitations
|
||||||
|
|
||||||
|
- The band comes from a trusted header until authentication arrives in v1.0.0.
|
||||||
|
Anyone who can reach the port can promote themselves to platinum.
|
||||||
|
- The tenant key is the session id, for the same reason.
|
||||||
|
- Displacement (dropping queued low-band work to make room) is not implemented;
|
||||||
|
the plan defers it, and bounded queues plus TTL cover most of the need.
|
||||||
|
|
||||||
## v0.2.0 — 2026-09-16
|
## v0.2.0 — 2026-09-16
|
||||||
|
|
||||||
The router serves traffic. `llmgw server` is real: it fronts a static set of
|
The router serves traffic. `llmgw server` is real: it fronts a static set of
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ MODULE := gitea.mynodes.xyz/ilker/llmgw
|
|||||||
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
|
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
|
||||||
LDFLAGS := -X '$(MODULE)/internal/version.Version=$(VERSION)'
|
LDFLAGS := -X '$(MODULE)/internal/version.Version=$(VERSION)'
|
||||||
|
|
||||||
.PHONY: build test race lint fmt clean rename
|
.PHONY: build test race lint fmt clean rename proto proto-tools
|
||||||
|
|
||||||
build:
|
build:
|
||||||
go build -ldflags "$(LDFLAGS)" -o bin/llmgw ./cmd/llmgw
|
go build -ldflags "$(LDFLAGS)" -o bin/llmgw ./cmd/llmgw
|
||||||
@@ -20,6 +20,17 @@ fmt:
|
|||||||
go fmt ./...
|
go fmt ./...
|
||||||
go vet ./...
|
go vet ./...
|
||||||
|
|
||||||
|
# Regenerate the agent protocol. Generated code is committed, so this only
|
||||||
|
# needs running when api/proto changes.
|
||||||
|
proto: proto-tools
|
||||||
|
buf lint
|
||||||
|
buf generate
|
||||||
|
|
||||||
|
proto-tools:
|
||||||
|
go install github.com/bufbuild/buf/cmd/buf@latest
|
||||||
|
go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
|
||||||
|
go install connectrpc.com/connect/cmd/protoc-gen-connect-go@latest
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -rf bin dist
|
rm -rf bin dist
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,18 @@ heterogeneous GPUs.
|
|||||||
It serves LLMs running on your own GPU servers (vLLM / SGLang) behind one
|
It serves LLMs running on your own GPU servers (vLLM / SGLang) behind one
|
||||||
OpenAI-compatible endpoint. Design document: [llmgw-plan.md](llmgw-plan.md).
|
OpenAI-compatible endpoint. Design document: [llmgw-plan.md](llmgw-plan.md).
|
||||||
|
|
||||||
|
## Getting started
|
||||||
|
|
||||||
|
Starting from nothing — no packages installed, no drivers, no nodes — follow
|
||||||
|
[docs/install.md](docs/install.md): it builds a control plane, adds a node and
|
||||||
|
serves a model, and says plainly which parts llmgw does not install for you.
|
||||||
|
|
||||||
|
| Guide | For |
|
||||||
|
|---|---|
|
||||||
|
| [docs/install.md](docs/install.md) | Standing up a cluster from scratch |
|
||||||
|
| [docs/node-onboarding.md](docs/node-onboarding.md) | Adding nodes to one that exists |
|
||||||
|
| [docs/onboarding.md](docs/onboarding.md) | Giving people accounts and keys |
|
||||||
|
|
||||||
## Status
|
## Status
|
||||||
|
|
||||||
Releases track the roadmap phases; see [CHANGELOG.md](CHANGELOG.md).
|
Releases track the roadmap phases; see [CHANGELOG.md](CHANGELOG.md).
|
||||||
@@ -14,14 +26,29 @@ Releases track the roadmap phases; see [CHANGELOG.md](CHANGELOG.md).
|
|||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| v0.1.0 | 0 | Skeleton and sim-engine | released |
|
| v0.1.0 | 0 | Skeleton and sim-engine | released |
|
||||||
| v0.2.0 | 1 | Static router (filter/score/pick, streaming proxy) | released |
|
| v0.2.0 | 1 | Static router (filter/score/pick, streaming proxy) | released |
|
||||||
| v0.3.0 | 1.5 | Admission control (priority bands) | planned |
|
| v0.3.0 | 1.5 | Admission control (priority bands) | released |
|
||||||
| v0.4.0 | 2 | Agent: GPU/IB discovery, single-node engines | planned |
|
| v0.4.0 | 2 | Agent: GPU/IB discovery, engine supervision | released |
|
||||||
| v0.5.0 | 3 | Control plane: catalog, scheduler, reconciler | planned |
|
| v0.5.0 | 3 | Control plane: catalog, placement, reconciler, ctl | released |
|
||||||
| v0.6.0 | 4 | Multi-node placement and fabric awareness | planned |
|
| v0.6.0 | 4 | Multi-node placement and fabric awareness | released |
|
||||||
| v1.0.0 | 5 | Tenancy, prefix affinity, benchmarking | planned |
|
| v1.0.0 | 5 | Tenancy, prefix affinity, benchmarking | planned |
|
||||||
|
|
||||||
As of v0.2.0 the router serves traffic. There is still no admission control, so
|
As of v0.6.0 a cluster runs unattended: agents report their hardware, the
|
||||||
a batch workload can starve interactive traffic — that is what v0.3.0 is for.
|
control plane places replicas — including ones larger than a single machine —
|
||||||
|
models load when someone asks for them and stop when nobody does, and keys,
|
||||||
|
budgets and rate limits decide who may use any of it.
|
||||||
|
|
||||||
|
Before v1.0.0: tracing, reference documentation, and filtering on `ctl nodes`
|
||||||
|
and `ctl replicas` for fleets too large to read a whole table of. Two items
|
||||||
|
that were on this list are not gaps: a model falls through to its fallbacks
|
||||||
|
when there is no capacity as well as when a budget is spent, and routing
|
||||||
|
profiles are chosen by the key rather than by a request header on purpose —
|
||||||
|
a header would let any client promote itself into the top priority band.
|
||||||
|
|
||||||
|
One thing is worth saying plainly:
|
||||||
|
multi-node replicas are proven on a simulated cluster and on real hardware
|
||||||
|
without a fabric. The InfiniBand, NVML and NCCL paths are written against the
|
||||||
|
formats and failure modes those tools actually have, but they have not yet run
|
||||||
|
on a machine that has them.
|
||||||
|
|
||||||
## Building
|
## Building
|
||||||
|
|
||||||
@@ -31,6 +58,24 @@ make test # unit tests
|
|||||||
make race # with the race detector
|
make race # with the race detector
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Operating a cluster
|
||||||
|
|
||||||
|
```sh
|
||||||
|
llmgw ctl nodes # capacity, health, what is running where
|
||||||
|
llmgw ctl models # catalog and replica counts
|
||||||
|
llmgw ctl replicas --model llama # replicas and their shards
|
||||||
|
llmgw ctl explain-placement llama # why a model has not been placed
|
||||||
|
llmgw ctl drain gpu-a-01 # move work off a node, one replica at a time
|
||||||
|
llmgw ctl cordon gpu-a-01 # stop new placement, leave what runs
|
||||||
|
```
|
||||||
|
|
||||||
|
`ctl` talks to the admin port (`:9090` by default), not the client-facing one.
|
||||||
|
Add `--json` to any command for machine-readable output.
|
||||||
|
|
||||||
|
`explain-placement` is the one worth knowing about: "unschedulable" on its own
|
||||||
|
says nothing about which of six constraints bit, so it prints every candidate
|
||||||
|
considered and the reason it was rejected.
|
||||||
|
|
||||||
## Running the router
|
## Running the router
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
@@ -51,6 +96,12 @@ For anything real use a config file; [deploy/examples/server.yaml](deploy/exampl
|
|||||||
is commented throughout. Metrics, health and pprof are on the admin port (`:9090`
|
is commented throughout. Metrics, health and pprof are on the admin port (`:9090`
|
||||||
by default), deliberately separate from the client-facing one.
|
by default), deliberately separate from the client-facing one.
|
||||||
|
|
||||||
|
Requests carry a priority band (`x-llmgw-band`). Under load, low bands wait at
|
||||||
|
the gateway while high bands keep flowing, so the cluster degrades from the
|
||||||
|
bottom up instead of everywhere at once. Measured effect: interactive p99 stays
|
||||||
|
within 1.24x of idle while 500 concurrent batch requests run, against 58x with
|
||||||
|
admission off.
|
||||||
|
|
||||||
Routing profiles select how endpoints are scored: `chat` favours session
|
Routing profiles select how endpoints are scored: `chat` favours session
|
||||||
affinity, `batch` favours the shortest queue, `long-context` favours free KV
|
affinity, `batch` favours the shortest queue, `long-context` favours free KV
|
||||||
cache. Pick one per model in the config, or per request with the
|
cache. Pick one per model in the config, or per request with the
|
||||||
|
|||||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,119 @@
|
|||||||
|
// Code generated by protoc-gen-connect-go. DO NOT EDIT.
|
||||||
|
//
|
||||||
|
// Source: llmgw/v1/agent.proto
|
||||||
|
|
||||||
|
// Package llmgw.v1 is the protocol between a node agent and the control plane
|
||||||
|
// (plan §5.6).
|
||||||
|
//
|
||||||
|
// The agent dials out and holds a bidirectional stream open. That direction is
|
||||||
|
// deliberate: GPU nodes sit behind firewalls and NAT far more often than a
|
||||||
|
// control plane does, and an outbound connection needs no inbound rule on every
|
||||||
|
// node in the fleet.
|
||||||
|
package llmgwv1connect
|
||||||
|
|
||||||
|
import (
|
||||||
|
connect "connectrpc.com/connect"
|
||||||
|
context "context"
|
||||||
|
errors "errors"
|
||||||
|
v1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||||
|
http "net/http"
|
||||||
|
strings "strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// This is a compile-time assertion to ensure that this generated file and the connect package are
|
||||||
|
// compatible. If you get a compiler error that this constant is not defined, this code was
|
||||||
|
// generated with a version of connect newer than the one compiled into your binary. You can fix the
|
||||||
|
// problem by either regenerating this code with an older version of connect or updating the connect
|
||||||
|
// version compiled into your binary.
|
||||||
|
const _ = connect.IsAtLeastVersion1_13_0
|
||||||
|
|
||||||
|
const (
|
||||||
|
// AgentServiceName is the fully-qualified name of the AgentService service.
|
||||||
|
AgentServiceName = "llmgw.v1.AgentService"
|
||||||
|
)
|
||||||
|
|
||||||
|
// These constants are the fully-qualified names of the RPCs defined in this package. They're
|
||||||
|
// exposed at runtime as Spec.Procedure and as the final two segments of the HTTP route.
|
||||||
|
//
|
||||||
|
// Note that these are different from the fully-qualified method names used by
|
||||||
|
// google.golang.org/protobuf/reflect/protoreflect. To convert from these constants to
|
||||||
|
// reflection-formatted method names, remove the leading slash and convert the remaining slash to a
|
||||||
|
// period.
|
||||||
|
const (
|
||||||
|
// AgentServiceConnectProcedure is the fully-qualified name of the AgentService's Connect RPC.
|
||||||
|
AgentServiceConnectProcedure = "/llmgw.v1.AgentService/Connect"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AgentServiceClient is a client for the llmgw.v1.AgentService service.
|
||||||
|
type AgentServiceClient interface {
|
||||||
|
// Connect opens the agent's stream. It stays open for the agent's lifetime;
|
||||||
|
// losing it is how each side learns the other is gone.
|
||||||
|
Connect(context.Context) *connect.BidiStreamForClient[v1.AgentMessage, v1.ControlMessage]
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewAgentServiceClient constructs a client for the llmgw.v1.AgentService service. By default, it
|
||||||
|
// uses the Connect protocol with the binary Protobuf Codec, asks for gzipped responses, and sends
|
||||||
|
// uncompressed requests. To use the gRPC or gRPC-Web protocols, supply the connect.WithGRPC() or
|
||||||
|
// connect.WithGRPCWeb() options.
|
||||||
|
//
|
||||||
|
// The URL supplied here should be the base URL for the Connect or gRPC server (for example,
|
||||||
|
// http://api.acme.com or https://acme.com/grpc).
|
||||||
|
func NewAgentServiceClient(httpClient connect.HTTPClient, baseURL string, opts ...connect.ClientOption) AgentServiceClient {
|
||||||
|
baseURL = strings.TrimRight(baseURL, "/")
|
||||||
|
agentServiceMethods := v1.File_llmgw_v1_agent_proto.Services().ByName("AgentService").Methods()
|
||||||
|
return &agentServiceClient{
|
||||||
|
connect: connect.NewClient[v1.AgentMessage, v1.ControlMessage](
|
||||||
|
httpClient,
|
||||||
|
baseURL+AgentServiceConnectProcedure,
|
||||||
|
connect.WithSchema(agentServiceMethods.ByName("Connect")),
|
||||||
|
connect.WithClientOptions(opts...),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// agentServiceClient implements AgentServiceClient.
|
||||||
|
type agentServiceClient struct {
|
||||||
|
connect *connect.Client[v1.AgentMessage, v1.ControlMessage]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Connect calls llmgw.v1.AgentService.Connect.
|
||||||
|
func (c *agentServiceClient) Connect(ctx context.Context) *connect.BidiStreamForClient[v1.AgentMessage, v1.ControlMessage] {
|
||||||
|
return c.connect.CallBidiStream(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AgentServiceHandler is an implementation of the llmgw.v1.AgentService service.
|
||||||
|
type AgentServiceHandler interface {
|
||||||
|
// Connect opens the agent's stream. It stays open for the agent's lifetime;
|
||||||
|
// losing it is how each side learns the other is gone.
|
||||||
|
Connect(context.Context, *connect.BidiStream[v1.AgentMessage, v1.ControlMessage]) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewAgentServiceHandler builds an HTTP handler from the service implementation. It returns the
|
||||||
|
// path on which to mount the handler and the handler itself.
|
||||||
|
//
|
||||||
|
// By default, handlers support the Connect, gRPC, and gRPC-Web protocols with the binary Protobuf
|
||||||
|
// and JSON codecs. They also support gzip compression.
|
||||||
|
func NewAgentServiceHandler(svc AgentServiceHandler, opts ...connect.HandlerOption) (string, http.Handler) {
|
||||||
|
agentServiceMethods := v1.File_llmgw_v1_agent_proto.Services().ByName("AgentService").Methods()
|
||||||
|
agentServiceConnectHandler := connect.NewBidiStreamHandler(
|
||||||
|
AgentServiceConnectProcedure,
|
||||||
|
svc.Connect,
|
||||||
|
connect.WithSchema(agentServiceMethods.ByName("Connect")),
|
||||||
|
connect.WithHandlerOptions(opts...),
|
||||||
|
)
|
||||||
|
return "/llmgw.v1.AgentService/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch r.URL.Path {
|
||||||
|
case AgentServiceConnectProcedure:
|
||||||
|
agentServiceConnectHandler.ServeHTTP(w, r)
|
||||||
|
default:
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnimplementedAgentServiceHandler returns CodeUnimplemented from all methods.
|
||||||
|
type UnimplementedAgentServiceHandler struct{}
|
||||||
|
|
||||||
|
func (UnimplementedAgentServiceHandler) Connect(context.Context, *connect.BidiStream[v1.AgentMessage, v1.ControlMessage]) error {
|
||||||
|
return connect.NewError(connect.CodeUnimplemented, errors.New("llmgw.v1.AgentService.Connect is not implemented"))
|
||||||
|
}
|
||||||
@@ -0,0 +1,312 @@
|
|||||||
|
syntax = "proto3";
|
||||||
|
|
||||||
|
// Package llmgw.v1 is the protocol between a node agent and the control plane
|
||||||
|
// (plan §5.6).
|
||||||
|
//
|
||||||
|
// The agent dials out and holds a bidirectional stream open. That direction is
|
||||||
|
// deliberate: GPU nodes sit behind firewalls and NAT far more often than a
|
||||||
|
// control plane does, and an outbound connection needs no inbound rule on every
|
||||||
|
// node in the fleet.
|
||||||
|
package llmgw.v1;
|
||||||
|
|
||||||
|
option go_package = "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1;llmgwv1";
|
||||||
|
|
||||||
|
service AgentService {
|
||||||
|
// Connect opens the agent's stream. It stays open for the agent's lifetime;
|
||||||
|
// losing it is how each side learns the other is gone.
|
||||||
|
rpc Connect(stream AgentMessage) returns (stream ControlMessage);
|
||||||
|
}
|
||||||
|
|
||||||
|
// AgentMessage is anything a node tells the control plane.
|
||||||
|
message AgentMessage {
|
||||||
|
oneof msg {
|
||||||
|
Register register = 1;
|
||||||
|
Heartbeat heartbeat = 2;
|
||||||
|
ShardStatus shard = 3;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ControlMessage is anything the control plane tells a node.
|
||||||
|
message ControlMessage {
|
||||||
|
oneof msg {
|
||||||
|
StartShard start = 1;
|
||||||
|
StopShard stop = 2;
|
||||||
|
DrainShard drain = 3;
|
||||||
|
Ack ack = 4;
|
||||||
|
RemoveWeights remove_weights = 5;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register is the first message on every stream. It carries the node's full
|
||||||
|
// hardware inventory, so the control plane needs no separate discovery call and
|
||||||
|
// a reconnecting agent automatically refreshes what the scheduler knows.
|
||||||
|
message Register {
|
||||||
|
string node_id = 1;
|
||||||
|
string hostname = 2;
|
||||||
|
// mgmt_addr is the address engines are reachable on, and the address NCCL
|
||||||
|
// bootstraps over. Never a fabric address: NCCL rendezvouses over TCP before
|
||||||
|
// it moves to RDMA.
|
||||||
|
string mgmt_addr = 3;
|
||||||
|
map<string, string> labels = 4;
|
||||||
|
|
||||||
|
repeated GPU gpus = 5;
|
||||||
|
// fabrics is empty on a node without InfiniBand. Such a node is its own
|
||||||
|
// single-node island and can never host a multi-node replica.
|
||||||
|
repeated FabricAttachment fabrics = 6;
|
||||||
|
|
||||||
|
string agent_version = 7;
|
||||||
|
// engine_versions decides which CLI flags and metric names apply; both drift
|
||||||
|
// between engine releases (plan §14).
|
||||||
|
map<string, string> engine_versions = 8;
|
||||||
|
|
||||||
|
// running_shards are the engines this agent is already supervising, including
|
||||||
|
// ones adopted from a previous agent.
|
||||||
|
//
|
||||||
|
// This is what lets a control plane restart recover instead of orphaning live
|
||||||
|
// engines. Without it the control plane forgets every replica, places new
|
||||||
|
// ones, and the old engines keep holding GPUs the inventory believes are free
|
||||||
|
// — so the new engines fail to allocate, with an error naming the wrong
|
||||||
|
// process (plan §6.5).
|
||||||
|
repeated RunningShard running_shards = 9;
|
||||||
|
|
||||||
|
// memory_total_mb is the machine's RAM, and 0 means the agent did not say —
|
||||||
|
// which is what every agent older than this field reports, and is treated as
|
||||||
|
// no constraint rather than as a node with no memory.
|
||||||
|
//
|
||||||
|
// Placement needs it because a CPU profile has no card to be free, so every
|
||||||
|
// ready node looks like a candidate: a 14B model whose working set is 24 GB
|
||||||
|
// would be placed just as happily on a node with 8 GB.
|
||||||
|
int64 memory_total_mb = 10;
|
||||||
|
|
||||||
|
// weights are the model files this node can see, from the directories its
|
||||||
|
// configuration names.
|
||||||
|
//
|
||||||
|
// How they got there is the operator's business and deliberately outside
|
||||||
|
// llmgw's: a shared filesystem, an image exported over the fabric, rsync, or
|
||||||
|
// a copy made by hand all look identical here. What matters to placement is
|
||||||
|
// only whether they are present, because a node without them either cannot
|
||||||
|
// start the replica at all or must first move tens of gigabytes.
|
||||||
|
//
|
||||||
|
// Empty means the node was not asked to look, which is not the same as having
|
||||||
|
// nothing — an agent with no model directories configured behaves exactly as
|
||||||
|
// every agent did before this existed, and constrains nothing.
|
||||||
|
repeated LocalWeights weights = 11;
|
||||||
|
|
||||||
|
// storage is the disk behind those weights, one entry per filesystem.
|
||||||
|
//
|
||||||
|
// Reported so an operator can see a node filling up before a fetch dies at
|
||||||
|
// ninety percent, and so placement can refuse a node that cannot hold a
|
||||||
|
// model rather than discovering it during the download.
|
||||||
|
//
|
||||||
|
// Empty means the node was not asked to look, exactly as for weights, and
|
||||||
|
// constrains nothing.
|
||||||
|
repeated Storage storage = 12;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Storage is one filesystem a node keeps weights on.
|
||||||
|
//
|
||||||
|
// Per filesystem rather than per configured directory: free space belongs to
|
||||||
|
// the disk, and two model directories on one disk are one pool of bytes.
|
||||||
|
message Storage {
|
||||||
|
// path is one directory on this filesystem, so the operator recognises it.
|
||||||
|
string path = 1;
|
||||||
|
// free_bytes is what a non-root process can write — not the root reserve,
|
||||||
|
// which is space the agent cannot have.
|
||||||
|
int64 free_bytes = 2;
|
||||||
|
int64 total_bytes = 3;
|
||||||
|
// shared marks a filesystem the node did not fill. Free space on one is
|
||||||
|
// shared with every other node using it, so two nodes each reporting 500 GB
|
||||||
|
// free on the same filer have 500 GB between them and not a terabyte.
|
||||||
|
bool shared = 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunningShard describes an engine already running on a node.
|
||||||
|
//
|
||||||
|
// It carries enough for the control plane to rebuild the replica it belongs to,
|
||||||
|
// which is more than a shard id: the replica's identity, its pool and fabric,
|
||||||
|
// and the leader endpoint clients reach it through.
|
||||||
|
message RunningShard {
|
||||||
|
string replica_id = 1;
|
||||||
|
string shard_id = 2;
|
||||||
|
int32 rank = 3;
|
||||||
|
int32 nnodes = 4;
|
||||||
|
repeated int32 gpu_indices = 5;
|
||||||
|
repeated string hcas = 6;
|
||||||
|
int32 http_port = 7;
|
||||||
|
ShardStatus.State state = 8;
|
||||||
|
|
||||||
|
// model and profile name the catalog entry this shard was started from.
|
||||||
|
string model = 9;
|
||||||
|
string profile = 10;
|
||||||
|
// pool and fabric are echoed back from the StartShard that created it; the
|
||||||
|
// agent has no way to derive them.
|
||||||
|
string pool = 11;
|
||||||
|
string fabric = 12;
|
||||||
|
|
||||||
|
string leader_addr = 13;
|
||||||
|
int32 leader_port = 14;
|
||||||
|
}
|
||||||
|
|
||||||
|
message GPU {
|
||||||
|
int32 index = 1;
|
||||||
|
string uuid = 2;
|
||||||
|
string name = 3;
|
||||||
|
int32 memory_total_mb = 4;
|
||||||
|
int32 memory_used_mb = 5;
|
||||||
|
string compute_capability = 6;
|
||||||
|
int32 numa_node = 7;
|
||||||
|
string pci_bus_id = 8;
|
||||||
|
string driver_version = 9;
|
||||||
|
string cuda_version = 10;
|
||||||
|
// nvlink_peers is only populated by agents built with the nvml tag.
|
||||||
|
repeated int32 nvlink_peers = 11;
|
||||||
|
// problems keeps a GPU out of placement: ECC faults, or memory already held
|
||||||
|
// by another process.
|
||||||
|
repeated string problems = 12;
|
||||||
|
int32 temperature_c = 13;
|
||||||
|
int32 ecc_errors = 14;
|
||||||
|
// utilization_percent is how busy the card was over the last sample period.
|
||||||
|
// Zero when the driver is too old to report it, which is not the same as
|
||||||
|
// idle: the metric is simply absent rather than wrong.
|
||||||
|
int32 utilization_percent = 15;
|
||||||
|
}
|
||||||
|
|
||||||
|
message FabricAttachment {
|
||||||
|
// fabric is the operator-assigned name. It is not derived from the GID
|
||||||
|
// prefix: OpenSM defaults to fe80:: on every fabric, so a matching prefix
|
||||||
|
// proves nothing (plan §5.2).
|
||||||
|
string fabric = 1;
|
||||||
|
repeated string hcas = 2;
|
||||||
|
int32 rate_gbps = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Heartbeat says the node is alive and reports anything that changed cheaply.
|
||||||
|
// LocalWeights is one model file or directory a node can see.
|
||||||
|
message LocalWeights {
|
||||||
|
// path is what an engine would be given, so it can be compared against a
|
||||||
|
// model's source without interpretation.
|
||||||
|
string path = 1;
|
||||||
|
int64 size_bytes = 2;
|
||||||
|
// shared marks weights on a filesystem the node did not have to fill —
|
||||||
|
// reported so a fabric with a shared filer is not mistaken for one where
|
||||||
|
// every node happens to have fetched its own copy.
|
||||||
|
bool shared = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Heartbeat {
|
||||||
|
string node_id = 1;
|
||||||
|
// gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on
|
||||||
|
// a card after registration.
|
||||||
|
repeated int32 gpu_memory_used_mb = 2;
|
||||||
|
repeated string new_problems = 3;
|
||||||
|
|
||||||
|
// storage for the same reason the GPU memory is here: it changes while the
|
||||||
|
// agent runs, and a figure sampled once at registration is a figure that is
|
||||||
|
// wrong on exactly the node worth watching — the one busy filling up.
|
||||||
|
//
|
||||||
|
// Always sent, because it is a few numbers per filesystem.
|
||||||
|
repeated Storage storage = 4;
|
||||||
|
|
||||||
|
// weights only when they changed, because the list is long enough that
|
||||||
|
// sending it every few seconds would be paying for nothing most of the time.
|
||||||
|
//
|
||||||
|
// weights_changed is what says the list is meant: without it an empty list
|
||||||
|
// would be ambiguous between "this node now holds nothing" and "nothing to
|
||||||
|
// report", and the first would quietly empty a node's inventory.
|
||||||
|
bool weights_changed = 5;
|
||||||
|
repeated LocalWeights weights = 6;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShardStatus reports a shard's lifecycle transition.
|
||||||
|
message ShardStatus {
|
||||||
|
string node_id = 1;
|
||||||
|
string replica_id = 2;
|
||||||
|
string shard_id = 3;
|
||||||
|
State state = 4;
|
||||||
|
string error = 5;
|
||||||
|
int32 exit_code = 6;
|
||||||
|
// last_output carries the tail of the engine log on a failure. When a model
|
||||||
|
// fails to load this is the only diagnosis available.
|
||||||
|
repeated string last_output = 7;
|
||||||
|
|
||||||
|
enum State {
|
||||||
|
STATE_UNSPECIFIED = 0;
|
||||||
|
STATE_PENDING = 1;
|
||||||
|
STATE_STARTING = 2;
|
||||||
|
STATE_LOADING = 3;
|
||||||
|
STATE_READY = 4;
|
||||||
|
STATE_DRAINING = 5;
|
||||||
|
STATE_STOPPED = 6;
|
||||||
|
STATE_FAILED = 7;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// StartShard launches one shard. It is idempotent by shard_id: the control
|
||||||
|
// plane retries, and a repeat must not produce a second process.
|
||||||
|
message StartShard {
|
||||||
|
string replica_id = 1;
|
||||||
|
string shard_id = 2;
|
||||||
|
|
||||||
|
int32 rank = 3;
|
||||||
|
int32 nnodes = 4;
|
||||||
|
string leader_addr = 5;
|
||||||
|
int32 leader_port = 6;
|
||||||
|
|
||||||
|
repeated int32 gpu_indices = 7;
|
||||||
|
// hcas are the adapters of the chosen fabric and nothing else. Handing a
|
||||||
|
// shard adapters from a second fabric lets NCCL pick one whose peer is
|
||||||
|
// unreachable, and the job hangs instead of failing (plan §5.3).
|
||||||
|
repeated string hcas = 8;
|
||||||
|
string mgmt_interface = 9;
|
||||||
|
int32 http_port = 10;
|
||||||
|
|
||||||
|
EngineProfile profile = 11;
|
||||||
|
int64 startup_timeout_ms = 12;
|
||||||
|
|
||||||
|
// pool and fabric are control plane concepts the agent cannot derive. They
|
||||||
|
// are sent so the agent can echo them back in RunningShard after a restart.
|
||||||
|
string pool = 13;
|
||||||
|
string fabric = 14;
|
||||||
|
}
|
||||||
|
|
||||||
|
message EngineProfile {
|
||||||
|
string name = 1;
|
||||||
|
string engine = 2;
|
||||||
|
string gpu_class = 3;
|
||||||
|
int32 tp = 4;
|
||||||
|
int32 pp = 5;
|
||||||
|
int32 gpus_per_node = 6;
|
||||||
|
int32 nodes = 7;
|
||||||
|
repeated string args = 8;
|
||||||
|
map<string, string> env = 9;
|
||||||
|
string model_path = 10;
|
||||||
|
string served_model_name = 11;
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveWeights deletes cached model files a node no longer needs.
|
||||||
|
//
|
||||||
|
// The paths are computed by the control plane from the catalog, but the agent
|
||||||
|
// checks them again before unlinking anything: it deletes only files inside its
|
||||||
|
// own cache directory whose names this system generated. A control plane that
|
||||||
|
// is wrong, compromised or simply newer than the agent must not be able to
|
||||||
|
// talk it into deleting a model an operator spent a week copying there.
|
||||||
|
message RemoveWeights {
|
||||||
|
repeated string paths = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// StopShard terminates a shard and frees its GPUs.
|
||||||
|
message StopShard {
|
||||||
|
string shard_id = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// DrainShard stops routing to a shard while letting in-flight work finish.
|
||||||
|
message DrainShard {
|
||||||
|
string shard_id = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ack confirms a registration or reports that a command could not be accepted.
|
||||||
|
message Ack {
|
||||||
|
string message_id = 1;
|
||||||
|
bool ok = 2;
|
||||||
|
string error = 3;
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
version: v2
|
||||||
|
managed:
|
||||||
|
enabled: true
|
||||||
|
plugins:
|
||||||
|
- local: protoc-gen-go
|
||||||
|
out: api/gen
|
||||||
|
opt: paths=source_relative
|
||||||
|
- local: protoc-gen-connect-go
|
||||||
|
out: api/gen
|
||||||
|
opt: paths=source_relative
|
||||||
|
inputs:
|
||||||
|
- directory: api/proto
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
version: v2
|
||||||
|
modules:
|
||||||
|
- path: api/proto
|
||||||
|
lint:
|
||||||
|
use:
|
||||||
|
- STANDARD
|
||||||
|
except:
|
||||||
|
# The standard names assume request/response pairs. This is one long-lived
|
||||||
|
# bidirectional stream carrying several kinds of message in each direction,
|
||||||
|
# so "AgentMessage" and "ControlMessage" say what they are and
|
||||||
|
# "ConnectRequest"/"ConnectResponse" would not.
|
||||||
|
- RPC_REQUEST_STANDARD_NAME
|
||||||
|
- RPC_RESPONSE_STANDARD_NAME
|
||||||
|
breaking:
|
||||||
|
use:
|
||||||
|
- FILE
|
||||||
Executable
+113
@@ -0,0 +1,113 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# add-node.sh — build a node and put it in the fleet, in one command.
|
||||||
|
#
|
||||||
|
# provision-node.sh and onboard-node.sh stay separate because the two halves
|
||||||
|
# need different access: one the hypervisor, one the certificate authority.
|
||||||
|
# That is a real boundary and this script does not erase it — it drives both
|
||||||
|
# where one person holds both, which is the common case and was the case here.
|
||||||
|
#
|
||||||
|
# The reason to have it is not keystrokes. The VM name, the hostname, the node
|
||||||
|
# id and the certificate's common name must all be the same string, and the
|
||||||
|
# address appears in three places; split across two invocations run minutes
|
||||||
|
# apart, that agreement is maintained by whoever is typing. Here it is stated
|
||||||
|
# once.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat >&2 <<'USAGE'
|
||||||
|
usage: add-node.sh --name NODE --ip ADDR/CIDR --server URL [options]
|
||||||
|
|
||||||
|
--name NODE node id, hostname and VM name; also the certificate's CN
|
||||||
|
--ip ADDR/CIDR e.g. 10.0.0.145/24
|
||||||
|
--server URL control plane, e.g. https://10.0.0.140:7443
|
||||||
|
|
||||||
|
--vmid N Proxmox VMID (default: the last octet of --ip)
|
||||||
|
--cores N (default 4)
|
||||||
|
--cpu TYPE QEMU CPU model (default host; see provision-node.sh)
|
||||||
|
--memory MB (default 8192)
|
||||||
|
--disk GB (default 20)
|
||||||
|
--fabric NAME InfiniBand fabric; repeatable as NAME:hca1,hca2.
|
||||||
|
Omit for a node with no fabric.
|
||||||
|
--llamacpp-bin P path to llama-server on the node
|
||||||
|
--vllm-bin P path to vllm
|
||||||
|
--sglang-python P path to the python that has sglang
|
||||||
|
--binary PATH llmgw binary to install (default: the one on PATH)
|
||||||
|
--pki DIR certificate authority (default /etc/llmgw/pki)
|
||||||
|
--ssh-user U (default ilker)
|
||||||
|
--provision-only build the VM and stop
|
||||||
|
--onboard-only skip provisioning; the machine already exists
|
||||||
|
USAGE
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
NAME= IP= SERVER= VMID= CORES=4 MEMORY=8192 DISK=20 CPU=
|
||||||
|
BINARY= PKI=/etc/llmgw/pki SSH_USER=ilker
|
||||||
|
PROVISION=1 ONBOARD=1
|
||||||
|
set -- "$@"
|
||||||
|
PASS_THROUGH=
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--name) NAME="$2"; shift 2;;
|
||||||
|
--ip) IP="$2"; shift 2;;
|
||||||
|
--server) SERVER="$2"; shift 2;;
|
||||||
|
--vmid) VMID="$2"; shift 2;;
|
||||||
|
--cores) CORES="$2"; shift 2;;
|
||||||
|
--cpu) CPU="$2"; shift 2;;
|
||||||
|
--memory) MEMORY="$2"; shift 2;;
|
||||||
|
--disk) DISK="$2"; shift 2;;
|
||||||
|
--binary) BINARY="$2"; shift 2;;
|
||||||
|
--pki) PKI="$2"; shift 2;;
|
||||||
|
--ssh-user) SSH_USER="$2"; shift 2;;
|
||||||
|
--provision-only) ONBOARD=0; shift;;
|
||||||
|
--onboard-only) PROVISION=0; shift;;
|
||||||
|
# Everything the onboarding step understands and this one does not need to.
|
||||||
|
--fabric|--llamacpp-bin|--vllm-bin|--sglang-python|--ib-root|--nvidia-smi|--mgmt-addr|--mgmt-if)
|
||||||
|
PASS_THROUGH="$PASS_THROUGH $1 $2"; shift 2;;
|
||||||
|
-h|--help) usage;;
|
||||||
|
*) echo "unknown option: $1" >&2; usage;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
[ -n "$NAME" ] && [ -n "$IP" ] && [ -n "$SERVER" ] || usage
|
||||||
|
|
||||||
|
# The address without its prefix, which is what onboarding and ssh want.
|
||||||
|
ADDR="${IP%%/*}"
|
||||||
|
# The last octet as the VMID by default: it keeps `qm list` and `ctl nodes` in
|
||||||
|
# the same order, and one fewer number to choose is one fewer to get wrong.
|
||||||
|
[ -n "$VMID" ] || VMID="${ADDR##*.}"
|
||||||
|
|
||||||
|
here() { dirname "$0"; }
|
||||||
|
PROVISION_SH="$(here)/provision-node.sh"
|
||||||
|
ONBOARD_SH="$(here)/onboard-node.sh"
|
||||||
|
[ -x "$PROVISION_SH" ] || PROVISION_SH="$(command -v llmgw-provision-node || true)"
|
||||||
|
[ -x "$ONBOARD_SH" ] || ONBOARD_SH="$(command -v llmgw-onboard-node || true)"
|
||||||
|
|
||||||
|
if [ "$PROVISION" = 1 ]; then
|
||||||
|
[ -x "$PROVISION_SH" ] || { echo "cannot find provision-node.sh" >&2; exit 1; }
|
||||||
|
echo "== 1/2 building $NAME as VM $VMID"
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
"$PROVISION_SH" --vmid "$VMID" --name "$NAME" --ip "$IP" \
|
||||||
|
--cores "$CORES" --memory "$MEMORY" --disk "$DISK" ${CPU:+--cpu "$CPU"}
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$ONBOARD" = 0 ]; then
|
||||||
|
echo "$NAME built. Onboard it from the CA host when you are ready."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
[ -x "$ONBOARD_SH" ] || { echo "cannot find onboard-node.sh" >&2; exit 1; }
|
||||||
|
echo
|
||||||
|
echo "== 2/2 onboarding $NAME"
|
||||||
|
# Onboarding waits for ssh itself, which is what makes running the two halves
|
||||||
|
# back to back safe: cloud-init is still regenerating host keys and restarting
|
||||||
|
# sshd for the first minute of a new machine's life.
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
"$ONBOARD_SH" --node "$NAME" --addr "$ADDR" --server "$SERVER" \
|
||||||
|
--pki "$PKI" --ssh-user "$SSH_USER" \
|
||||||
|
${BINARY:+--binary "$BINARY"} $PASS_THROUGH
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "Check it with: llmgw ctl nodes"
|
||||||
|
echo
|
||||||
|
echo "A node is not finished until the engines its profiles name are installed"
|
||||||
|
echo "and the weights are on disk. Onboarding checks that a configured engine"
|
||||||
|
echo "runs, but it installs neither — see docs/node-onboarding.md."
|
||||||
Executable
+270
@@ -0,0 +1,270 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
#
|
||||||
|
# Give a machine an identity in an llmgw cluster.
|
||||||
|
#
|
||||||
|
# Run on the host that holds the certificate authority, because that is the one
|
||||||
|
# thing that cannot be delegated: this script issues the credential by which the
|
||||||
|
# control plane will believe the node is who it says it is.
|
||||||
|
#
|
||||||
|
# What it does, in the order it matters:
|
||||||
|
# 1. issue a client certificate whose common name is the node id
|
||||||
|
# 2. copy the CA root, the certificate and its key to the machine
|
||||||
|
# 3. trust the root system-wide, so other tooling there stops complaining
|
||||||
|
# 4. install the binary, the configuration and the unit
|
||||||
|
# 5. start the agent
|
||||||
|
#
|
||||||
|
# The node id, the VM name, the hostname and the certificate common name are all
|
||||||
|
# the same string. They do not have to be, and every time they are not, someone
|
||||||
|
# spends an afternoon working out which of the four a message is talking about.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat >&2 <<USAGE
|
||||||
|
usage: $0 --node ID --addr HOST --server URL [options]
|
||||||
|
|
||||||
|
--node ID node id, hostname and certificate common name
|
||||||
|
--addr HOST how to reach the machine over ssh
|
||||||
|
--server URL the control plane's agent endpoint, e.g. https://10.0.0.140:7443
|
||||||
|
--fabric NAME InfiniBand fabric this node is attached to; repeatable as
|
||||||
|
--fabric NAME:hca1,hca2 . Omit for a node with no fabric,
|
||||||
|
which can host single-node replicas and will never be given
|
||||||
|
a multi-node one.
|
||||||
|
--mgmt-addr ADDR address engines are reachable on (default: --addr)
|
||||||
|
--mgmt-if NAME interface NCCL bootstraps over (default eth0)
|
||||||
|
--llamacpp-bin P path to llama-server
|
||||||
|
--vllm-bin P path to the vllm entry point
|
||||||
|
--sglang-python P python interpreter of the SGLang virtualenv
|
||||||
|
--nvidia-smi PATH where nvidia-smi lives, when it is not on PATH. Containerized
|
||||||
|
driver installs routinely put it somewhere else.
|
||||||
|
--ib-root DIR the sysfs InfiniBand directory (default /sys/class/infiniband)
|
||||||
|
--pki DIR where the authority lives (default /etc/llmgw/pki)
|
||||||
|
--binary PATH llmgw binary to install (default: the one running this).
|
||||||
|
Releases carry static linux/amd64 and linux/arm64 builds;
|
||||||
|
downloading one is better than copying whatever happens to
|
||||||
|
be on the machine you are running this from.
|
||||||
|
--ssh-user NAME (default ilker)
|
||||||
|
USAGE
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
NODE= ADDR= SERVER= MGMT= MGMT_IF=eth0
|
||||||
|
NVIDIA_SMI= IB_ROOT=/sys/class/infiniband
|
||||||
|
LLAMACPP_BIN= VLLM_BIN= SGLANG_PYTHON=
|
||||||
|
SSH_WAIT=180
|
||||||
|
PKI=/etc/llmgw/pki BINARY= SSH_USER=ilker
|
||||||
|
FABRIC_NAME= FABRIC_HCAS="mlx5_0,mlx5_1"
|
||||||
|
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--node) NODE="$2"; shift 2;;
|
||||||
|
--addr) ADDR="$2"; shift 2;;
|
||||||
|
--server) SERVER="$2"; shift 2;;
|
||||||
|
--fabric) FABRIC_NAME="${2%%:*}"; [ "${2#*:}" != "$2" ] && FABRIC_HCAS="${2#*:}"; shift 2;;
|
||||||
|
--mgmt-addr) MGMT="$2"; shift 2;;
|
||||||
|
--mgmt-if) MGMT_IF="$2"; shift 2;;
|
||||||
|
--llamacpp-bin) LLAMACPP_BIN="$2"; shift 2;;
|
||||||
|
--vllm-bin) VLLM_BIN="$2"; shift 2;;
|
||||||
|
--sglang-python) SGLANG_PYTHON="$2"; shift 2;;
|
||||||
|
--nvidia-smi) NVIDIA_SMI="$2"; shift 2;;
|
||||||
|
--ib-root) IB_ROOT="$2"; shift 2;;
|
||||||
|
--pki) PKI="$2"; shift 2;;
|
||||||
|
--binary) BINARY="$2"; shift 2;;
|
||||||
|
--ssh-user) SSH_USER="$2"; shift 2;;
|
||||||
|
*) usage;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
[ -n "$NODE" ] && [ -n "$ADDR" ] && [ -n "$SERVER" ] || usage
|
||||||
|
MGMT="${MGMT:-$ADDR}"
|
||||||
|
BINARY="${BINARY:-$(command -v llmgw)}"
|
||||||
|
[ -x "$BINARY" ] || { echo "no llmgw binary; pass --binary" >&2; exit 1; }
|
||||||
|
[ -f "$PKI/ca-key.pem" ] || { echo "no authority in $PKI; run 'llmgw ctl cert init-ca' first" >&2; exit 1; }
|
||||||
|
|
||||||
|
# accept-new, not "no": a machine provisioned a minute ago has a host key
|
||||||
|
# nobody has seen, and refusing it stopped onboarding dead with "Host key
|
||||||
|
# verification failed" — an error about ssh, in the middle of a script whose
|
||||||
|
# whole job is to talk to a new machine. accept-new learns an unknown key and
|
||||||
|
# still refuses a changed one, which is the distinction that matters: the risk
|
||||||
|
# being guarded against is a substituted machine, not a new one.
|
||||||
|
SSHOPTS="-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new"
|
||||||
|
SSH="ssh $SSHOPTS $SSH_USER@$ADDR"
|
||||||
|
say() { printf ' %s\n' "$*"; }
|
||||||
|
|
||||||
|
echo "onboarding $NODE at $ADDR"
|
||||||
|
|
||||||
|
# Waiting rather than failing: cloud-init restarts sshd after it regenerates
|
||||||
|
# the host keys, so a machine that answered a moment ago refuses connections
|
||||||
|
# for a few seconds. Observed while onboarding node05 — the agent was already
|
||||||
|
# registered and healthy while ssh was refusing.
|
||||||
|
wait_for_ssh() {
|
||||||
|
deadline=$(( $(date +%s) + SSH_WAIT ))
|
||||||
|
until $SSH true 2>/dev/null; do
|
||||||
|
[ "$(date +%s)" -lt "$deadline" ] || {
|
||||||
|
echo "cannot reach $SSH_USER@$ADDR over ssh after ${SSH_WAIT}s" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
}
|
||||||
|
wait_for_ssh
|
||||||
|
|
||||||
|
# 1. The certificate. Issued fresh every time: re-onboarding a machine should
|
||||||
|
# not depend on finding an old key, and the old one keeps working until it
|
||||||
|
# is revoked, which is a separate decision.
|
||||||
|
say "issuing a certificate"
|
||||||
|
TMP="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$TMP"' EXIT
|
||||||
|
"$BINARY" ctl cert issue-client "$NODE" --dir "$PKI" --out "$TMP" >/dev/null
|
||||||
|
|
||||||
|
# 2 and 3. Identity, and trusting the root so future tooling on that machine
|
||||||
|
# does not treat our certificates as strangers.
|
||||||
|
say "installing the certificate and trusting the root"
|
||||||
|
scp -q $SSHOPTS "$PKI/ca.pem" "$TMP/$NODE.pem" "$TMP/$NODE-key.pem" "$SSH_USER@$ADDR:/tmp/"
|
||||||
|
$SSH "set -e
|
||||||
|
sudo install -d -m 0755 /etc/llmgw
|
||||||
|
sudo install -m 0644 /tmp/ca.pem /etc/llmgw/ca.pem
|
||||||
|
sudo install -m 0644 /tmp/$NODE.pem /etc/llmgw/agent.pem
|
||||||
|
sudo install -m 0600 /tmp/$NODE-key.pem /etc/llmgw/agent-key.pem
|
||||||
|
sudo chown root:root /etc/llmgw/*
|
||||||
|
sudo install -m 0644 /tmp/ca.pem /usr/local/share/ca-certificates/llmgw-ca.crt
|
||||||
|
sudo update-ca-certificates >/dev/null 2>&1 || true
|
||||||
|
rm -f /tmp/ca.pem /tmp/$NODE.pem /tmp/$NODE-key.pem"
|
||||||
|
|
||||||
|
# 4. The binary, the configuration and the unit.
|
||||||
|
say "installing the binary"
|
||||||
|
scp -q $SSHOPTS "$BINARY" "$SSH_USER@$ADDR:/tmp/llmgw"
|
||||||
|
$SSH 'sudo install -m 0755 /tmp/llmgw /usr/local/bin/llmgw && rm -f /tmp/llmgw'
|
||||||
|
|
||||||
|
say "writing the configuration"
|
||||||
|
FABRICS="fabrics: []"
|
||||||
|
if [ -n "$FABRIC_NAME" ]; then
|
||||||
|
FABRICS="fabrics:
|
||||||
|
- name: $FABRIC_NAME
|
||||||
|
hcas: [$(echo "$FABRIC_HCAS" | sed 's/,/, /g')]"
|
||||||
|
fi
|
||||||
|
cat > "$TMP/agent.yaml" <<YAML
|
||||||
|
# $NODE
|
||||||
|
#
|
||||||
|
# Written by onboard-node.sh. Fabric membership is configuration and there is no
|
||||||
|
# way around it: OpenSM's default subnet prefix is the same on every fabric, so
|
||||||
|
# a matching prefix proves nothing and a wrong guess produces a replica that
|
||||||
|
# starts, passes health checks and then hangs on its first all-reduce.
|
||||||
|
node_id: $NODE
|
||||||
|
server: "$SERVER"
|
||||||
|
|
||||||
|
tls:
|
||||||
|
ca: /etc/llmgw/ca.pem
|
||||||
|
cert: /etc/llmgw/agent.pem
|
||||||
|
key: /etc/llmgw/agent-key.pem
|
||||||
|
|
||||||
|
mgmt_addr: "$MGMT"
|
||||||
|
mgmt_interface: $MGMT_IF
|
||||||
|
|
||||||
|
heartbeat_interval: 5s
|
||||||
|
reconnect_base: 1s
|
||||||
|
reconnect_max: 30s
|
||||||
|
|
||||||
|
$FABRICS
|
||||||
|
|
||||||
|
discovery:
|
||||||
|
# Empty means look on PATH. Worth setting where the driver lives outside the
|
||||||
|
# default paths, which containerized installs routinely do.
|
||||||
|
nvidia_smi: "$NVIDIA_SMI"
|
||||||
|
ib_root: $IB_ROOT
|
||||||
|
prefer_nvml: false
|
||||||
|
skip_gpus: false
|
||||||
|
|
||||||
|
# Where the engines live. Empty means "on PATH", which is rarely true: a node
|
||||||
|
# that cannot find its engine registers healthy and then fails every shard
|
||||||
|
# placed on it with "executable file not found".
|
||||||
|
engines:
|
||||||
|
llamacpp:
|
||||||
|
bin: "$LLAMACPP_BIN"
|
||||||
|
parallel: 4
|
||||||
|
vllm:
|
||||||
|
bin: "$VLLM_BIN"
|
||||||
|
launch_mode: ray
|
||||||
|
sglang:
|
||||||
|
python: "$SGLANG_PYTHON"
|
||||||
|
|
||||||
|
supervisor:
|
||||||
|
work_dir: /var/lib/llmgw-agent
|
||||||
|
log_lines: 512
|
||||||
|
stop_timeout: 60s
|
||||||
|
health_interval: 2s
|
||||||
|
YAML
|
||||||
|
scp -q "$TMP/agent.yaml" "$SSH_USER@$ADDR:/tmp/agent.yaml"
|
||||||
|
|
||||||
|
cat > "$TMP/llmgw-agent.service" <<'UNIT'
|
||||||
|
[Unit]
|
||||||
|
Description=llmgw node agent
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
ExecStart=/usr/local/bin/llmgw agent --config /etc/llmgw/agent.yaml
|
||||||
|
# Root, because the agent launches engines and needs device access. Engines
|
||||||
|
# outlive a restart of this unit and are adopted again on the next start, so
|
||||||
|
# KillMode must not take the whole process group with it.
|
||||||
|
KillMode=process
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=2s
|
||||||
|
StateDirectory=llmgw-agent
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
UNIT
|
||||||
|
scp -q "$TMP/llmgw-agent.service" "$SSH_USER@$ADDR:/tmp/"
|
||||||
|
|
||||||
|
# An engine path that is wrong, or right but unrunnable, is the failure this
|
||||||
|
# node type is most likely to have, and it is invisible: the agent registers
|
||||||
|
# healthy and every shard placed on it dies with "executable file not found"
|
||||||
|
# or a missing shared library. Checked here because it is cheap here and
|
||||||
|
# expensive later — llama-server on node05 was present, executable, and could
|
||||||
|
# not start for want of libgomp1, and the node reported ready throughout.
|
||||||
|
check_engine() {
|
||||||
|
[ -n "$2" ] || return 0
|
||||||
|
if $SSH "test -x '$2'" 2>/dev/null; then
|
||||||
|
if $SSH "'$2' --version >/dev/null 2>&1 || '$2' --help >/dev/null 2>&1"; then
|
||||||
|
say "$1: $2 runs"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
ENGINE_PROBLEMS="$ENGINE_PROBLEMS
|
||||||
|
$1 at $2 exists but will not run. Usually a missing shared library:
|
||||||
|
ssh $SSH_USER@$ADDR ldd $2 | grep 'not found'"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
ENGINE_PROBLEMS="$ENGINE_PROBLEMS
|
||||||
|
$1 is configured at $2, which is not an executable on $NODE."
|
||||||
|
}
|
||||||
|
ENGINE_PROBLEMS=
|
||||||
|
if [ -n "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then
|
||||||
|
say "checking the engines"
|
||||||
|
check_engine llamacpp "$LLAMACPP_BIN"
|
||||||
|
check_engine vllm "$VLLM_BIN"
|
||||||
|
check_engine sglang "$SGLANG_PYTHON"
|
||||||
|
fi
|
||||||
|
|
||||||
|
say "starting the agent"
|
||||||
|
$SSH 'set -e
|
||||||
|
sudo install -m 0644 -o root -g root /tmp/agent.yaml /etc/llmgw/agent.yaml
|
||||||
|
sudo install -m 0644 /tmp/llmgw-agent.service /etc/systemd/system/llmgw-agent.service
|
||||||
|
rm -f /tmp/agent.yaml /tmp/llmgw-agent.service
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
sudo systemctl enable --now llmgw-agent >/dev/null
|
||||||
|
sudo systemctl restart llmgw-agent'
|
||||||
|
|
||||||
|
echo
|
||||||
|
if [ -z "$LLAMACPP_BIN$VLLM_BIN$SGLANG_PYTHON" ]; then
|
||||||
|
say "no engine paths given; this node can only run the 'sim' engine until"
|
||||||
|
say "one is installed and the configuration points at it"
|
||||||
|
fi
|
||||||
|
if [ -n "$ENGINE_PROBLEMS" ]; then
|
||||||
|
echo "$NODE is onboarded but an engine will not run:$ENGINE_PROBLEMS"
|
||||||
|
echo
|
||||||
|
echo "The node will register as ready regardless. Fix this before a replica"
|
||||||
|
echo "is placed on it, or the shard fails and the reason is three logs away."
|
||||||
|
echo
|
||||||
|
fi
|
||||||
|
echo "$NODE onboarded. It should appear in 'llmgw ctl nodes' within a few seconds."
|
||||||
|
echo "If it does not, the control plane's log says why — a certificate whose"
|
||||||
|
echo "common name does not match the claimed node id is refused by design."
|
||||||
Executable
+96
@@ -0,0 +1,96 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
#
|
||||||
|
# Create a VM on Proxmox for a new llmgw node.
|
||||||
|
#
|
||||||
|
# Run on the Proxmox host. It does the half of onboarding that is about
|
||||||
|
# hardware; onboard-node.sh does the half that is about identity, and the two
|
||||||
|
# are separate because they are done by different people with different access.
|
||||||
|
#
|
||||||
|
# Idempotent only in the sense that it refuses to touch an existing VMID. A node
|
||||||
|
# that already exists is not something a provisioning script should have an
|
||||||
|
# opinion about.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat >&2 <<USAGE
|
||||||
|
usage: $0 --vmid N --name NAME --ip ADDR [options]
|
||||||
|
|
||||||
|
--vmid N Proxmox VM id; must be free
|
||||||
|
--name NAME VM name; use the node id, so the machine, the hostname and
|
||||||
|
the certificate common name are the same string
|
||||||
|
--ip ADDR address with prefix, e.g. 10.0.0.144/24
|
||||||
|
--gw ADDR gateway (default 10.0.0.1)
|
||||||
|
--cores N (default 4)
|
||||||
|
--cpu TYPE QEMU CPU model (default host)
|
||||||
|
--memory MB (default 8192)
|
||||||
|
--disk GB (default 20)
|
||||||
|
--bridge NAME (default vmbr0)
|
||||||
|
--storage NAME (default local-lvm)
|
||||||
|
--image PATH cloud image (default Ubuntu 24.04 in the ISO store)
|
||||||
|
--user NAME cloud-init user (default ilker)
|
||||||
|
--sshkeys PATH authorized_keys to install (default /root/.ssh/authorized_keys)
|
||||||
|
USAGE
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
|
||||||
|
VMID= NAME= IP= GW=10.0.0.1 CORES=4 MEMORY=8192 DISK=20
|
||||||
|
# host, not Proxmox's kvm64 default. kvm64 hides AVX, AVX2, FMA and even
|
||||||
|
# SSE4.2 from the guest, and an inference node without vector instructions is
|
||||||
|
# not slightly slower, it is unusable: llama.cpp falls back to its scalar
|
||||||
|
# backend and a 1.5B model dropped from 39 tokens a second to 5.8, with prompt
|
||||||
|
# processing 19x worse. It cost an afternoon to find because nothing is broken
|
||||||
|
# — every number is simply small.
|
||||||
|
#
|
||||||
|
# The trade is live migration: a VM pinned to the host CPU cannot move to a
|
||||||
|
# machine with a different one. For a node whose job is arithmetic, that is
|
||||||
|
# the right way round.
|
||||||
|
CPU=host
|
||||||
|
BRIDGE=vmbr0 STORAGE=local-lvm USER=ilker
|
||||||
|
IMAGE=/var/lib/vz/template/iso/noble-server-cloudimg-amd64.img
|
||||||
|
SSHKEYS=/root/.ssh/authorized_keys
|
||||||
|
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--vmid) VMID="$2"; shift 2;;
|
||||||
|
--name) NAME="$2"; shift 2;;
|
||||||
|
--ip) IP="$2"; shift 2;;
|
||||||
|
--gw) GW="$2"; shift 2;;
|
||||||
|
--cores) CORES="$2"; shift 2;;
|
||||||
|
--cpu) CPU="$2"; shift 2;;
|
||||||
|
--memory) MEMORY="$2"; shift 2;;
|
||||||
|
--disk) DISK="$2"; shift 2;;
|
||||||
|
--bridge) BRIDGE="$2"; shift 2;;
|
||||||
|
--storage) STORAGE="$2"; shift 2;;
|
||||||
|
--image) IMAGE="$2"; shift 2;;
|
||||||
|
--user) USER="$2"; shift 2;;
|
||||||
|
--sshkeys) SSHKEYS="$2"; shift 2;;
|
||||||
|
*) usage;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
[ -n "$VMID" ] && [ -n "$NAME" ] && [ -n "$IP" ] || usage
|
||||||
|
|
||||||
|
if qm status "$VMID" >/dev/null 2>&1; then
|
||||||
|
echo "VM $VMID already exists. Pick a free id; this script will not adopt one." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
[ -f "$IMAGE" ] || { echo "no cloud image at $IMAGE" >&2; exit 1; }
|
||||||
|
|
||||||
|
echo "creating $NAME as VM $VMID at $IP"
|
||||||
|
qm create "$VMID" \
|
||||||
|
--name "$NAME" --cores "$CORES" --memory "$MEMORY" --cpu "$CPU" \
|
||||||
|
--net0 "virtio,bridge=$BRIDGE" --ostype l26 --agent enabled=1
|
||||||
|
|
||||||
|
# Imported rather than installed: a cloud image plus cloud-init is the whole
|
||||||
|
# reason a node takes a minute to build instead of an afternoon.
|
||||||
|
qm importdisk "$VMID" "$IMAGE" "$STORAGE" >/dev/null
|
||||||
|
qm set "$VMID" --scsihw virtio-scsi-pci --scsi0 "$STORAGE:vm-$VMID-disk-0,discard=on,ssd=1" >/dev/null
|
||||||
|
qm disk resize "$VMID" scsi0 "${DISK}G" >/dev/null
|
||||||
|
qm set "$VMID" --ide2 "$STORAGE:cloudinit" --boot order=scsi0 >/dev/null
|
||||||
|
qm set "$VMID" \
|
||||||
|
--ipconfig0 "ip=$IP,gw=$GW" \
|
||||||
|
--nameserver "$GW 8.8.8.8" \
|
||||||
|
--ciuser "$USER" \
|
||||||
|
--sshkeys "$SSHKEYS" >/dev/null
|
||||||
|
|
||||||
|
qm start "$VMID"
|
||||||
|
echo "started. cloud-init takes a minute; then run onboard-node.sh from the CA host."
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
# llmgw agent configuration, one per GPU node.
|
||||||
|
#
|
||||||
|
# The agent dials out to the control plane and keeps one stream open. Engines
|
||||||
|
# keep serving if the agent restarts: the router talks to them directly, and
|
||||||
|
# orphan adoption picks them up again on the next start.
|
||||||
|
|
||||||
|
node_id: gpu-a-01 # defaults to the hostname
|
||||||
|
server: "http://llmgw-server.internal:7443"
|
||||||
|
|
||||||
|
# The agent's half of mutual TLS. The certificate's common name must be this
|
||||||
|
# node's id, because the control plane compares the two and refuses the stream
|
||||||
|
# when they disagree:
|
||||||
|
#
|
||||||
|
# llmgw ctl cert issue-client gpu-a-01 --dir /etc/llmgw/pki
|
||||||
|
#
|
||||||
|
# With this set, `server` above must be an https:// address — an agent holding
|
||||||
|
# certificates and dialling http:// is refused at startup rather than connecting
|
||||||
|
# in the clear without noticing.
|
||||||
|
#
|
||||||
|
# tls:
|
||||||
|
# ca: /etc/llmgw/ca.pem
|
||||||
|
# cert: /etc/llmgw/agent.pem
|
||||||
|
# key: /etc/llmgw/agent-key.pem
|
||||||
|
|
||||||
|
# The address engines are reachable on, and the address NCCL bootstraps over.
|
||||||
|
# Never a fabric address: NCCL rendezvouses over TCP before moving to RDMA.
|
||||||
|
mgmt_addr: "10.0.0.11"
|
||||||
|
mgmt_interface: eth0
|
||||||
|
|
||||||
|
labels:
|
||||||
|
rack: r12
|
||||||
|
|
||||||
|
heartbeat_interval: 5s
|
||||||
|
reconnect_base: 1s
|
||||||
|
reconnect_max: 30s
|
||||||
|
|
||||||
|
# Fabric labelling is explicit and there is no way around it. OpenSM's default
|
||||||
|
# subnet prefix is fe80:: on every fabric, so a matching prefix proves nothing
|
||||||
|
# and a wrong guess produces a replica that starts, passes health checks and
|
||||||
|
# then hangs on its first all-reduce (plan §5.2).
|
||||||
|
#
|
||||||
|
# A node with no fabrics is its own single-node island: it can host single-node
|
||||||
|
# replicas and will never be given a multi-node one.
|
||||||
|
fabrics:
|
||||||
|
- name: ib-a
|
||||||
|
hcas: [mlx5_0, mlx5_1, mlx5_2, mlx5_3]
|
||||||
|
rate_gbps: 200
|
||||||
|
|
||||||
|
discovery:
|
||||||
|
# Worth setting where the driver lives outside the default paths, which
|
||||||
|
# containerized driver installs often do.
|
||||||
|
nvidia_smi: ""
|
||||||
|
ib_root: /sys/class/infiniband
|
||||||
|
# NVML is the only source of NVLink topology, which is what keeps a
|
||||||
|
# tensor-parallel group off the PCIe path inside a node. It needs a binary
|
||||||
|
# built with `-tags nvml`; otherwise this silently falls back to nvidia-smi.
|
||||||
|
prefer_nvml: true
|
||||||
|
skip_gpus: false
|
||||||
|
|
||||||
|
# Where this node keeps model weights.
|
||||||
|
#
|
||||||
|
# Unset by default, and unset means the node is not asked to look — which is
|
||||||
|
# not the same as having nothing, and constrains nothing. Storage differs at
|
||||||
|
# every installation and llmgw deliberately does not own it: a Lustre mount, an
|
||||||
|
# image exported over the fabric with SRP or NVMe-oF, local NVMe, or a copy
|
||||||
|
# made by hand all look identical from here.
|
||||||
|
#
|
||||||
|
# What it buys is placement preferring a node that already holds the model over
|
||||||
|
# one that must first move tens of gigabytes. That matters most where fabrics
|
||||||
|
# are isolated from one another, because weights in one fabric are not weights
|
||||||
|
# in another and the only path between them is the management network.
|
||||||
|
#
|
||||||
|
# model_dirs:
|
||||||
|
# - /var/lib/llmgw-models
|
||||||
|
# - /mnt/lustre/models
|
||||||
|
|
||||||
|
engines:
|
||||||
|
vllm:
|
||||||
|
bin: /opt/venvs/vllm/bin/vllm
|
||||||
|
# Which multi-node mechanism works depends on the vLLM version, and the
|
||||||
|
# flags have changed more than once. Check the version you deploy.
|
||||||
|
launch_mode: ray # "ray" or "mp"
|
||||||
|
ray_bin: /opt/venvs/vllm/bin/ray
|
||||||
|
sglang:
|
||||||
|
# SGLang is launched as a module, which is its documented entry point.
|
||||||
|
python: /opt/venvs/sglang/bin/python
|
||||||
|
|
||||||
|
supervisor:
|
||||||
|
work_dir: /var/lib/llmgw-agent
|
||||||
|
# Engine output is the only diagnosis when a model fails to load, so keep a
|
||||||
|
# useful tail of it.
|
||||||
|
log_lines: 512
|
||||||
|
# Engines need time to free GPU memory cleanly before being killed.
|
||||||
|
stop_timeout: 60s
|
||||||
|
health_interval: 2s
|
||||||
+203
-15
@@ -7,7 +7,91 @@
|
|||||||
# __ for nesting: LLMGW_ROUTER__RETRIES=3
|
# __ for nesting: LLMGW_ROUTER__RETRIES=3
|
||||||
|
|
||||||
listen: ":8080" # client-facing OpenAI API
|
listen: ":8080" # client-facing OpenAI API
|
||||||
admin_listen: ":9090" # metrics, health, pprof — do not expose this publicly
|
admin_listen: "127.0.0.1:9090" # dashboard, management API, pprof — keep it local
|
||||||
|
# Metrics only, on their own port, so Prometheus can reach them without pprof
|
||||||
|
# being reachable too. Empty means off; metrics stay on admin_listen either way.
|
||||||
|
# metrics_listen: "0.0.0.0:9091"
|
||||||
|
# Where node agents connect. Separate from the client port so the fleet-facing
|
||||||
|
# surface can be firewalled independently.
|
||||||
|
agent_listen: ":7443"
|
||||||
|
|
||||||
|
# Mutual TLS on the agent port (plan §5.6). With it, a node's identity is taken
|
||||||
|
# from its certificate's common name and an agent claiming a different node id
|
||||||
|
# is refused. Without it the port is plaintext HTTP/2 and anyone who can reach
|
||||||
|
# it can claim to be any node, so it must stay on a trusted network.
|
||||||
|
#
|
||||||
|
# These certificates come from llmgw's own authority, not a public one: public
|
||||||
|
# authorities stopped issuing the clientAuth usage, so no Let's Encrypt
|
||||||
|
# certificate can authenticate an agent.
|
||||||
|
#
|
||||||
|
# llmgw ctl cert init-ca --dir /etc/llmgw/pki
|
||||||
|
# llmgw ctl cert issue-server llmgw-ctl --host 10.0.0.140 --dir /etc/llmgw/pki
|
||||||
|
#
|
||||||
|
# Every address an agent might dial has to appear in --host, bare IP addresses
|
||||||
|
# included: hostname verification reads subject alternative names and ignores
|
||||||
|
# the common name entirely.
|
||||||
|
#
|
||||||
|
# This covers the fleet-facing port only. The client-facing API above is public
|
||||||
|
# and belongs behind a publicly trusted certificate.
|
||||||
|
#
|
||||||
|
# agent_tls:
|
||||||
|
# ca: /etc/llmgw/ca.pem
|
||||||
|
# cert: /etc/llmgw/server.pem
|
||||||
|
# key: /etc/llmgw/server-key.pem
|
||||||
|
|
||||||
|
# The public page at "/". Someone who reaches this address without a key needs a
|
||||||
|
# route to getting one, and the gateway cannot guess who to send them to.
|
||||||
|
#
|
||||||
|
# landing:
|
||||||
|
# contact: "platform@example.com" # or an https:// URL, or "#platform on Slack"
|
||||||
|
# notice: "Internal use only."
|
||||||
|
|
||||||
|
# API keys on the client-facing port (plan §7.9). Off by default, because a
|
||||||
|
# gateway on a trusted network with no keys issued still has to serve — but
|
||||||
|
# nothing else protects that port, so turning it on is what makes it safe to
|
||||||
|
# expose.
|
||||||
|
#
|
||||||
|
# llmgw keys init --store /var/lib/llmgw/llmgw.db
|
||||||
|
# llmgw keys create --store /var/lib/llmgw/llmgw.db --project default "laptop"
|
||||||
|
#
|
||||||
|
# The key is printed once. Only its hash is stored, so there is nothing to
|
||||||
|
# recover it from.
|
||||||
|
#
|
||||||
|
# auth:
|
||||||
|
# enabled: true
|
||||||
|
# store: /var/lib/llmgw/llmgw.db
|
||||||
|
# # How long a resolved key is trusted before being read again. This is the
|
||||||
|
# # price of keeping the database off the request path: it bounds how long a
|
||||||
|
# # revoked key keeps working.
|
||||||
|
# cache_ttl: 30s
|
||||||
|
# # How long a rejected key is remembered as rejected, so a client looping
|
||||||
|
# # with a bad key cannot become a database read per request.
|
||||||
|
# negative_ttl: 5s
|
||||||
|
#
|
||||||
|
# # Serve the signup form. What it creates is a pending account that can do
|
||||||
|
# # nothing until `llmgw users approve` places it in a project, so this is not
|
||||||
|
# # a way in — it only moves key creation from you to the person who needs one.
|
||||||
|
# # Off, there is no form and you issue keys with `llmgw keys create --claim`.
|
||||||
|
# signup_open: true
|
||||||
|
#
|
||||||
|
# # What each model costs, in micro-units per thousand tokens. A unit is
|
||||||
|
# # whatever you decide: self-hosted models have no per-token price, so it can
|
||||||
|
# # be GPU-seconds, a notional rate, or 1000 everywhere — which turns budgets
|
||||||
|
# # into plain token quotas.
|
||||||
|
# #
|
||||||
|
# # A model priced at zero is exempt from budget checks, which is what lets a
|
||||||
|
# # key with nothing left still be served a cheap fallback.
|
||||||
|
# default_price: { prompt_per_1k: 0, completion_per_1k: 0 }
|
||||||
|
# pricing:
|
||||||
|
# llama-3.3-70b: { prompt_per_1k: 1000, completion_per_1k: 3000 }
|
||||||
|
# qwen2.5-32b: { prompt_per_1k: 400, completion_per_1k: 1200 }
|
||||||
|
#
|
||||||
|
# # Usage rows are written in batches off the request path. Beyond buffer_size
|
||||||
|
# # they are dropped rather than blocking a request: losing some accounting is
|
||||||
|
# # bad, turning a stalled database into an outage is worse.
|
||||||
|
# flush_interval: 200ms
|
||||||
|
# batch_size: 500
|
||||||
|
# buffer_size: 8192
|
||||||
|
|
||||||
router:
|
router:
|
||||||
scrape_interval: 2s
|
scrape_interval: 2s
|
||||||
@@ -29,6 +113,39 @@ router:
|
|||||||
failure_threshold: 3
|
failure_threshold: 3
|
||||||
|
|
||||||
default_profile: chat # chat | batch | long-context | debug
|
default_profile: chat # chat | batch | long-context | debug
|
||||||
|
|
||||||
|
# Approximate prefix routing (plan §7.5 tier 1): send a request to the replica
|
||||||
|
# that has probably already prefilled its prefix.
|
||||||
|
#
|
||||||
|
# It earns its keep when the working set does not fit in one replica's cache.
|
||||||
|
# With a handful of prompts and a large cache, scattering is free — every
|
||||||
|
# replica ends up holding everything — and this only adds constraint. Measured
|
||||||
|
# with sim-engines whose caches were too small for the working set: cache hits
|
||||||
|
# went from 22.5% to 81.7%.
|
||||||
|
#
|
||||||
|
# Watch llmgw_prefix_decisions_total to tune it. A high
|
||||||
|
# load_balance_override count means the guard is doing its job and affinity is
|
||||||
|
# not getting a chance; a high cache_miss_min_load means match_ratio or
|
||||||
|
# min_match is too strict for your traffic.
|
||||||
|
prefix:
|
||||||
|
enabled: false
|
||||||
|
# The shortest match worth acting on, in bytes. Below this the overlap is
|
||||||
|
# coincidence — two English prompts share "The".
|
||||||
|
min_match: 64
|
||||||
|
# The share of the prompt that must match. A short shared header on a very
|
||||||
|
# long prompt saves almost no prefill.
|
||||||
|
match_ratio: 0.3
|
||||||
|
# The imbalance guard, and both must trip before affinity is abandoned.
|
||||||
|
# Absolute alone fires on an idle cluster where one replica has two requests
|
||||||
|
# and another none; relative alone fires on a busy one where 200 against 180
|
||||||
|
# is a rounding error. Without the guard the cache hit rate rises while p99
|
||||||
|
# collapses.
|
||||||
|
balance_abs: 8
|
||||||
|
balance_rel: 1.5
|
||||||
|
# An engine evicts on its own schedule, so an old record may describe a
|
||||||
|
# cache that is gone.
|
||||||
|
ttl: 10m
|
||||||
|
max_nodes: 100000
|
||||||
session_header: x-session-id # affinity key; falls back to the `user` field
|
session_header: x-session-id # affinity key; falls back to the `user` field
|
||||||
# Ask engines for usage even when the client did not, so token accounting
|
# Ask engines for usage even when the client did not, so token accounting
|
||||||
# works regardless of client behaviour. The extra chunk is dropped on the way
|
# works regardless of client behaviour. The extra chunk is dropped on the way
|
||||||
@@ -43,6 +160,32 @@ router:
|
|||||||
idle: 60s # between streamed tokens; catches a wedged engine
|
idle: 60s # between streamed tokens; catches a wedged engine
|
||||||
total: 30m
|
total: 30m
|
||||||
|
|
||||||
|
# Flow control: decides *when* a request may proceed, before scheduling
|
||||||
|
# decides where it goes. Without it a batch workload starves interactive
|
||||||
|
# traffic, because an engine queue is FIFO and knows nothing about tenants.
|
||||||
|
admission:
|
||||||
|
enabled: true
|
||||||
|
# With auth on, the key decides the band and this header is ignored. With
|
||||||
|
# auth off the header is trusted, so any client can promote itself into the
|
||||||
|
# top band — do not expose the port in that configuration.
|
||||||
|
band_header: x-llmgw-band
|
||||||
|
default_band: gold
|
||||||
|
# A band waits once pool saturation passes its ceiling, so the cluster
|
||||||
|
# degrades from the bottom up. A ceiling of 1.0 means "flow up to the
|
||||||
|
# healthy limit"; use a value above 1 for a band that must never wait.
|
||||||
|
bands:
|
||||||
|
- { name: platinum, priority: 400, saturation_ceiling: 1.00 }
|
||||||
|
- { name: gold, priority: 300, saturation_ceiling: 0.85 }
|
||||||
|
- { name: silver, priority: 200, saturation_ceiling: 0.60 }
|
||||||
|
- { name: bronze, priority: 100, saturation_ceiling: 0.30 }
|
||||||
|
# Drop a request that has waited this long: the client has usually given up,
|
||||||
|
# and sending the GPU work nobody is waiting for helps no one.
|
||||||
|
queue_ttl: 30s
|
||||||
|
# Reject immediately once a band's queue is this deep, rather than letting
|
||||||
|
# it accumulate silently.
|
||||||
|
queue_capacity_per_band: 2000
|
||||||
|
interval: 10ms
|
||||||
|
|
||||||
# What the scorers normalize against. Measure these with `ctl bench` rather
|
# What the scorers normalize against. Measure these with `ctl bench` rather
|
||||||
# than guessing: target_waiting in particular governs how the whole system
|
# than guessing: target_waiting in particular governs how the whole system
|
||||||
# behaves under load. Too low starves the GPU, too high brings back queueing
|
# behaves under load. Too low starves the GPU, too high brings back queueing
|
||||||
@@ -53,27 +196,72 @@ router:
|
|||||||
kv_ceiling: 0.9
|
kv_ceiling: 0.9
|
||||||
session_load_factor: 1.25
|
session_load_factor: 1.25
|
||||||
|
|
||||||
|
# How aggressively the control plane reconciles.
|
||||||
|
control:
|
||||||
|
interval: 5s
|
||||||
|
base_backoff: 10s
|
||||||
|
max_backoff: 10m
|
||||||
|
# Node expiry is staged deliberately: a network blip must not tear down
|
||||||
|
# replicas that are still serving, and rebuilding a multi-node replica costs
|
||||||
|
# tens of minutes of model loading.
|
||||||
|
unreachable_after: 15s
|
||||||
|
lost_after: 60s
|
||||||
|
port_range: [20000, 20999]
|
||||||
|
mgmt_interface: eth0
|
||||||
|
|
||||||
|
# Each model needs a source — a weights path or a HuggingFace id — and at least
|
||||||
|
# one profile if the control plane is to place it. Without profiles the model is
|
||||||
|
# catalogued but never started, which is the router-only mode below.
|
||||||
models:
|
models:
|
||||||
- name: llama-3.3-70b
|
- name: llama-3.3-70b
|
||||||
|
source: meta-llama/Llama-3.3-70B-Instruct
|
||||||
aliases: [llama]
|
aliases: [llama]
|
||||||
fallbacks: [qwen2.5-32b]
|
fallbacks: [qwen2.5-32b]
|
||||||
routing_profile: chat
|
routing_profile: chat
|
||||||
|
scaling: { min: 1, max: 4 }
|
||||||
|
profiles:
|
||||||
|
# Two nodes of four H100s each. A multi-node profile requires a fabric,
|
||||||
|
# and a replica never spans two of them.
|
||||||
|
- name: tp8
|
||||||
|
engine: vllm
|
||||||
|
gpu_class: h100-80g
|
||||||
|
tp: 8
|
||||||
|
nodes: 2
|
||||||
|
gpus_per_node: 4
|
||||||
|
priority: 100
|
||||||
|
weight: 8
|
||||||
|
startup_timeout: 30m
|
||||||
|
# Where a whole node is free, one node is cheaper to schedule and faster
|
||||||
|
# to start than two.
|
||||||
|
- name: tp4-a100
|
||||||
|
engine: vllm
|
||||||
|
gpu_class: a100-40g
|
||||||
|
tp: 4
|
||||||
|
priority: 50
|
||||||
|
weight: 3
|
||||||
|
startup_timeout: 20m
|
||||||
|
|
||||||
- name: qwen2.5-32b
|
- name: qwen2.5-32b
|
||||||
|
source: Qwen/Qwen2.5-32B-Instruct
|
||||||
aliases: [qwen]
|
aliases: [qwen]
|
||||||
routing_profile: batch
|
routing_profile: batch
|
||||||
|
scaling: { min: 1, max: 2 }
|
||||||
|
profiles:
|
||||||
|
- name: tp2
|
||||||
|
engine: sglang
|
||||||
|
gpu_class: h100-80g
|
||||||
|
tp: 2
|
||||||
|
priority: 100
|
||||||
|
weight: 2
|
||||||
|
startup_timeout: 15m
|
||||||
|
|
||||||
endpoints:
|
# A static replica list, used only when no model declares a profile. That is the
|
||||||
- id: gpu-a-01
|
# router-only mode for testing against sim-engines; with the control plane
|
||||||
url: http://gpu-a-01:8000
|
# running, agents provide the replicas instead.
|
||||||
models: [llama-3.3-70b]
|
#
|
||||||
pool: ib-a/h100-80g
|
# endpoints:
|
||||||
weight: 100 # from ctl bench; do not estimate it by hand
|
# - id: sim-1
|
||||||
priority: 100 # higher pools are tried first
|
# url: http://127.0.0.1:8001
|
||||||
|
# models: [llama-3.3-70b]
|
||||||
- id: gpu-b-01
|
# pool: local
|
||||||
url: http://gpu-b-01:8000
|
# weight: 1
|
||||||
models: [llama-3.3-70b, qwen2.5-32b]
|
|
||||||
pool: ib-b/a100-80g
|
|
||||||
weight: 45
|
|
||||||
priority: 50
|
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
# Scraping llmgw.
|
||||||
|
#
|
||||||
|
# Point this at metrics_listen, not admin_listen. They are different ports for
|
||||||
|
# a reason: the admin listener also carries pprof, which can dump the process
|
||||||
|
# heap, and API keys are in that heap in plaintext.
|
||||||
|
#
|
||||||
|
# In the control plane's configuration:
|
||||||
|
#
|
||||||
|
# admin_listen: "127.0.0.1:9090" # dashboard, management API, pprof
|
||||||
|
# metrics_listen: "0.0.0.0:9091" # metrics only, safe to reach
|
||||||
|
#
|
||||||
|
# One target for the whole fleet. Node and GPU figures are exported by the
|
||||||
|
# control plane, which already holds them — agents report hardware at
|
||||||
|
# registration and state on every heartbeat — so there is nothing to scrape on
|
||||||
|
# the nodes and no Prometheus configuration per machine.
|
||||||
|
global:
|
||||||
|
scrape_interval: 15s
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: llmgw
|
||||||
|
static_configs:
|
||||||
|
- targets: ["llmgw-ctl.internal:9091"]
|
||||||
|
labels:
|
||||||
|
cluster: primary
|
||||||
|
|
||||||
|
# Worth alerting on, in rough order of how often it matters.
|
||||||
|
#
|
||||||
|
# A node that has stopped reporting:
|
||||||
|
# llmgw_node_last_seen_seconds > 60
|
||||||
|
#
|
||||||
|
# A model that should be serving and is not. Covers a replica that will never
|
||||||
|
# start as well as one that has died, which look the same from a distance:
|
||||||
|
# llmgw_model_replicas_ready == 0
|
||||||
|
#
|
||||||
|
# A card kept out of placement by a fault, which is otherwise invisible until
|
||||||
|
# a model will not fit:
|
||||||
|
# llmgw_gpu_healthy == 0
|
||||||
|
#
|
||||||
|
# A fleet holding models nobody is asking for. Memory used says a model is
|
||||||
|
# loaded; utilisation says whether it is doing anything:
|
||||||
|
# avg_over_time(llmgw_gpu_utilization_percent[1h]) < 5
|
||||||
|
#
|
||||||
|
# Requests queueing because the cluster is full, which is the pair of figures
|
||||||
|
# that needed two systems to correlate before these were on one endpoint:
|
||||||
|
# llmgw_admission_queue_depth > 0
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
# systemd units
|
||||||
|
|
||||||
|
Two roles, two units. Copy the one you need to `/etc/systemd/system/`, put the
|
||||||
|
matching configuration in `/etc/llmgw/`, and `systemctl enable --now` it.
|
||||||
|
|
||||||
|
## The control plane — `llmgw-server.service`
|
||||||
|
|
||||||
|
Runs unprivileged. It needs to read its own TLS key, so that file must belong to
|
||||||
|
the `llmgw` user and nothing else:
|
||||||
|
|
||||||
|
```
|
||||||
|
useradd --system --home-dir /var/lib/llmgw --shell /usr/sbin/nologin llmgw
|
||||||
|
chown llmgw:llmgw /etc/llmgw/server-key.pem
|
||||||
|
chmod 0400 /etc/llmgw/server-key.pem
|
||||||
|
```
|
||||||
|
|
||||||
|
`StateDirectory=llmgw` creates `/var/lib/llmgw` with the right owner, which is
|
||||||
|
where the store belongs. `ProtectSystem=strict` makes the rest of the filesystem
|
||||||
|
read-only, so anything else it writes to has to be named explicitly.
|
||||||
|
|
||||||
|
## The agent — `llmgw-agent.service`
|
||||||
|
|
||||||
|
Runs as root, because it launches engines and needs device access.
|
||||||
|
|
||||||
|
`KillMode=process` is the line that matters and the reason this file exists
|
||||||
|
rather than being generated. Without it, systemd kills the whole control group
|
||||||
|
on stop — including the engines, which hold a loaded model and are serving
|
||||||
|
traffic. Engines are meant to outlive an agent restart: the router talks to them
|
||||||
|
directly, and the agent adopts them again when it comes back. Taking them down
|
||||||
|
with the agent turns a thirty-second upgrade into a full model reload on every
|
||||||
|
node.
|
||||||
|
|
||||||
|
systemd will log `Found left-over process ... in control group while starting
|
||||||
|
unit. Ignoring.` on the next start. That is the mechanism working, not a fault.
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=llmgw node agent
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
ExecStart=/usr/local/bin/llmgw agent --config /etc/llmgw/agent.yaml
|
||||||
|
# Root, because the agent launches engines and needs device access. Engines
|
||||||
|
# outlive a restart of this unit and are adopted again on the next start, so
|
||||||
|
# KillMode must not take the whole process group with it.
|
||||||
|
KillMode=process
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=2s
|
||||||
|
StateDirectory=llmgw-agent
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=llmgw control plane
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
ExecStart=/usr/local/bin/llmgw server --config /etc/llmgw/server.yaml
|
||||||
|
User=llmgw
|
||||||
|
Group=llmgw
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=2s
|
||||||
|
NoNewPrivileges=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
PrivateTmp=true
|
||||||
|
StateDirectory=llmgw
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
+753
@@ -0,0 +1,753 @@
|
|||||||
|
# Installing llmgw from nothing
|
||||||
|
|
||||||
|
Written for whoever is standing up the cluster: you have servers, you have root
|
||||||
|
on them, and nothing is installed yet. By the end there is a control plane, a
|
||||||
|
node in the fleet, and a model answering requests.
|
||||||
|
|
||||||
|
It assumes Debian or Ubuntu and adjusts nothing for other distributions beyond
|
||||||
|
the package names. Everything here has been done on a real cluster except where
|
||||||
|
this page says otherwise, and where it says otherwise it says so plainly.
|
||||||
|
|
||||||
|
## What llmgw does not install
|
||||||
|
|
||||||
|
Read this first, because it is the difference between an afternoon and a week.
|
||||||
|
|
||||||
|
llmgw runs engines and routes requests. It does **not** install GPU drivers,
|
||||||
|
CUDA, inference engines, or model weights. A node missing any of them still
|
||||||
|
registers, still reports `ready`, and still fails every replica placed on it —
|
||||||
|
so the useful order is to make a node able to serve *before* onboarding it, not
|
||||||
|
after.
|
||||||
|
|
||||||
|
| You provide | llmgw provides |
|
||||||
|
| --- | --- |
|
||||||
|
| NVIDIA driver and `nvidia-smi` | GPU discovery, classes, health |
|
||||||
|
| InfiniBand stack (`rdma-core`/OFED) | Fabric discovery and the invariant that a replica never spans one |
|
||||||
|
| vLLM, SGLang or llama.cpp | Launching them, the right flags, supervision |
|
||||||
|
| Model weights on disk | Placement, routing, accounting |
|
||||||
|
|
||||||
|
The reason is boring and unavoidable: drivers and engines are platform decisions
|
||||||
|
with kernel modules and version matrices attached, and distributing weights is
|
||||||
|
its own problem — hundreds of gigabytes that want a shared filesystem, not a
|
||||||
|
copy from somebody's laptop. It is a known gap, not an oversight.
|
||||||
|
|
||||||
|
## 0. Before you begin
|
||||||
|
|
||||||
|
Four machines is a reasonable first cluster: one control plane and three nodes.
|
||||||
|
One of each also works.
|
||||||
|
|
||||||
|
**The control plane** needs no GPU. It is a single process holding a SQLite
|
||||||
|
file; 2 GB of RAM and a few gigabytes of disk is generous. It must be reachable
|
||||||
|
from every node on the agent port, and from your users on the client port.
|
||||||
|
|
||||||
|
**Nodes** need whatever the model needs, plus about 8 GB for the operating
|
||||||
|
system and the agent. They dial the control plane; the control plane never dials
|
||||||
|
them, so nodes need no inbound access except your own SSH.
|
||||||
|
|
||||||
|
**Ports**
|
||||||
|
|
||||||
|
| Port | Who connects | Expose to |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| 8080 | clients, the web UI | your users, usually via a reverse proxy |
|
||||||
|
| 7443 | node agents | the node network only |
|
||||||
|
| 9090 | `llmgw ctl`, metrics, pprof | loopback only — it serves pprof, which can dump the process heap, and API keys live in that heap |
|
||||||
|
|
||||||
|
Decide now which address nodes will use to reach the control plane. It goes into
|
||||||
|
a certificate and changing it later means reissuing one.
|
||||||
|
|
||||||
|
## 1. The binary
|
||||||
|
|
||||||
|
One static binary, no runtime dependencies, the same on every machine.
|
||||||
|
|
||||||
|
```
|
||||||
|
VERSION=v0.6.0 # check the releases page for the current tag
|
||||||
|
curl -fsSLO https://gitea.mynodes.xyz/ilker/llmgw/releases/download/$VERSION/llmgw-$VERSION-linux-amd64
|
||||||
|
curl -fsSL https://gitea.mynodes.xyz/ilker/llmgw/releases/download/$VERSION/checksums.txt | sha256sum -c --ignore-missing
|
||||||
|
sudo install -m0755 llmgw-$VERSION-linux-amd64 /usr/local/bin/llmgw
|
||||||
|
llmgw --version
|
||||||
|
```
|
||||||
|
|
||||||
|
`arm64` and `darwin` builds are published alongside. They are built with
|
||||||
|
`CGO_ENABLED=0` deliberately, so a binary built today runs on a node whose libc
|
||||||
|
is older than the builder's.
|
||||||
|
|
||||||
|
To build instead of download, you need Go 1.26 or later and nothing else:
|
||||||
|
|
||||||
|
```
|
||||||
|
git clone https://gitea.mynodes.xyz/ilker/llmgw && cd llmgw
|
||||||
|
CGO_ENABLED=0 go build -o llmgw ./cmd/llmgw
|
||||||
|
```
|
||||||
|
|
||||||
|
## 2. The certificate authority
|
||||||
|
|
||||||
|
Agents authenticate to the control plane with mutual TLS, and a node's identity
|
||||||
|
is taken from its certificate's common name. A public authority cannot supply
|
||||||
|
these: Let's Encrypt and its peers stopped issuing certificates carrying the
|
||||||
|
`clientAuth` usage, so nothing they hand out can authenticate an agent.
|
||||||
|
|
||||||
|
Run this wherever you intend to keep the authority. It talks to nothing and
|
||||||
|
works on files, so it does not have to be a cluster machine — and there is an
|
||||||
|
argument that it should not be.
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo llmgw ctl cert init-ca --dir /etc/llmgw/pki
|
||||||
|
```
|
||||||
|
|
||||||
|
That writes `ca.pem` and `ca-key.pem`. **Whoever holds `ca-key.pem` can mint any
|
||||||
|
node's identity**, so it stays here; `ca.pem` is copied to every machine and is
|
||||||
|
not a secret.
|
||||||
|
|
||||||
|
Now the control plane's own certificate. Every address an agent might dial must
|
||||||
|
appear in `--host`, bare IP addresses included — hostname verification reads
|
||||||
|
subject alternative names and ignores the common name entirely, so a certificate
|
||||||
|
naming the host only in its subject is rejected by every client:
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo llmgw ctl cert issue-server llmgw-ctl \
|
||||||
|
--host 10.0.0.140 --host llmgw-ctl.internal \
|
||||||
|
--dir /etc/llmgw/pki --out /etc/llmgw
|
||||||
|
```
|
||||||
|
|
||||||
|
This covers the fleet-facing port only. The client-facing API is public and
|
||||||
|
belongs behind an ordinary publicly trusted certificate, usually terminated by a
|
||||||
|
reverse proxy in front of port 8080.
|
||||||
|
|
||||||
|
## 3. The control plane
|
||||||
|
|
||||||
|
### The user and directories
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo useradd --system --home /var/lib/llmgw --shell /usr/sbin/nologin llmgw
|
||||||
|
sudo install -d -o llmgw -g llmgw -m 0750 /var/lib/llmgw
|
||||||
|
sudo install -d -m 0755 /etc/llmgw
|
||||||
|
sudo chown root:llmgw /etc/llmgw/ca.pem /etc/llmgw/server.pem /etc/llmgw/server-key.pem
|
||||||
|
sudo chmod 0640 /etc/llmgw/server-key.pem
|
||||||
|
```
|
||||||
|
|
||||||
|
### The configuration
|
||||||
|
|
||||||
|
`/etc/llmgw/server.yaml`. This is the smallest file that is not a toy: mutual
|
||||||
|
TLS on, authentication on, one model, and limits that apply to anyone nobody has
|
||||||
|
given limits to.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
listen: ":8080"
|
||||||
|
admin_listen: "127.0.0.1:9090"
|
||||||
|
agent_listen: ":7443"
|
||||||
|
|
||||||
|
agent_tls:
|
||||||
|
ca: /etc/llmgw/ca.pem
|
||||||
|
cert: /etc/llmgw/server.pem
|
||||||
|
key: /etc/llmgw/server-key.pem
|
||||||
|
|
||||||
|
auth:
|
||||||
|
enabled: true
|
||||||
|
store: /var/lib/llmgw/llmgw.db
|
||||||
|
signup_open: true
|
||||||
|
# Applied to any key no scope gives a limit to. Without these, a project
|
||||||
|
# created without limits is metered by nothing at all, and the first sign of
|
||||||
|
# that is the bill or the outage.
|
||||||
|
default_rpm: 60
|
||||||
|
default_tpm: 200000
|
||||||
|
|
||||||
|
landing:
|
||||||
|
contact: "platform@example.com"
|
||||||
|
|
||||||
|
# Start with none. Models are added while the cluster runs — see step 7 —
|
||||||
|
# and anything listed here is declarative: it is in the catalog on every start
|
||||||
|
# and cannot be removed without editing this file.
|
||||||
|
models: []
|
||||||
|
```
|
||||||
|
|
||||||
|
`admin_listen` is on loopback on purpose. It serves pprof, which can dump the
|
||||||
|
process heap, and API keys sit in that heap in plaintext. To see the dashboard
|
||||||
|
from elsewhere, promote an account and use `/admin` on the client port — do not
|
||||||
|
move this listener.
|
||||||
|
|
||||||
|
The store file is created on first start; there is no separate migration step.
|
||||||
|
|
||||||
|
### The service
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo cp deploy/systemd/llmgw-server.service /etc/systemd/system/
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
sudo systemctl enable --now llmgw-server
|
||||||
|
systemctl status llmgw-server
|
||||||
|
```
|
||||||
|
|
||||||
|
### Your account
|
||||||
|
|
||||||
|
Accounts are created and approved from a shell on this machine, deliberately:
|
||||||
|
having a shell here is what makes someone an administrator, and there are no
|
||||||
|
roles to get subtly wrong.
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo -u llmgw llmgw users create you@example.com --password 'change me now'
|
||||||
|
# prints a user id, usr-xxxxxxxx
|
||||||
|
sudo -u llmgw llmgw users promote usr-xxxxxxxx
|
||||||
|
```
|
||||||
|
|
||||||
|
`promote` is what grants `/admin`. Sign in at `http://<control-plane>:8080/login`,
|
||||||
|
change the password, and issue yourself a key from the profile page.
|
||||||
|
|
||||||
|
### Check it
|
||||||
|
|
||||||
|
```
|
||||||
|
curl -s -o /dev/null -w '%{http_code}\n' http://127.0.0.1:8080/health # 200, with an empty body
|
||||||
|
sudo -u llmgw llmgw ctl status
|
||||||
|
```
|
||||||
|
|
||||||
|
`ctl status` will report no nodes and no models, which is correct — nothing has
|
||||||
|
been added yet. It is worth running now so its output is familiar before you
|
||||||
|
need it.
|
||||||
|
|
||||||
|
## 4. Preparing a node
|
||||||
|
|
||||||
|
**Do this before onboarding.** A node missing its driver or engine registers
|
||||||
|
healthy and fails on first placement, and the reason is three logs away.
|
||||||
|
|
||||||
|
### The driver
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y nvidia-driver-550 # or whatever your cards need
|
||||||
|
sudo reboot
|
||||||
|
nvidia-smi # must print your GPUs
|
||||||
|
```
|
||||||
|
|
||||||
|
`nvidia-smi` must be on `PATH` for the user the agent runs as, which is root.
|
||||||
|
Containerised driver installs often put it elsewhere; if so, note the path and
|
||||||
|
set `discovery.nvidia_smi` in the agent configuration rather than moving it.
|
||||||
|
|
||||||
|
A node with no GPU at all is fine and is not an error. It reports zero cards and
|
||||||
|
can host CPU profiles, which is a good way to try llmgw before committing
|
||||||
|
hardware to it.
|
||||||
|
|
||||||
|
### NVLink, if your GPUs have it
|
||||||
|
|
||||||
|
The published binary cannot see NVLink topology. NVML is the only source of it,
|
||||||
|
its bindings need cgo, and the portable binary is built without cgo on purpose
|
||||||
|
so it runs on a node whose libc is older than the builder's.
|
||||||
|
|
||||||
|
Without that topology, a tensor-parallel group takes the lowest free GPU
|
||||||
|
indices. On an SXM board with NVSwitch that is correct, because every card is
|
||||||
|
equidistant. On a PCIe box where NVLink bridges pair specific cards, a group
|
||||||
|
straddling two bridges falls back to PCIe and runs several times slower — and
|
||||||
|
nothing reports it, because from the outside it is simply a replica that is
|
||||||
|
slow.
|
||||||
|
|
||||||
|
For those nodes, install the `-nvml` build published beside the portable one:
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo install -m0755 llmgw-$VERSION-linux-amd64-nvml /usr/local/bin/llmgw
|
||||||
|
```
|
||||||
|
|
||||||
|
It is dynamically linked, so it needs a libc no older than the builder's and the
|
||||||
|
NVIDIA driver present. Check it took effect with:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw ctl nodes --json | grep -c nvlink_peers # 0 means the topology is unknown
|
||||||
|
```
|
||||||
|
|
||||||
|
`llmgw_gpu_nvlink_peers` in the metrics says the same thing across a fleet: a
|
||||||
|
flat zero everywhere means the topology is unknown rather than absent.
|
||||||
|
|
||||||
|
### InfiniBand, if you have it
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo apt-get install -y rdma-core ibverbs-utils infiniband-diags
|
||||||
|
ibstat # must list your HCAs with State: Active
|
||||||
|
ls /sys/class/infiniband
|
||||||
|
```
|
||||||
|
|
||||||
|
The agent checks what you declare against what the node has. An adapter that is
|
||||||
|
not there, or that has no ACTIVE port — physically linked but with no subnet
|
||||||
|
manager, which survives a reboot and looks like a software problem — takes that
|
||||||
|
fabric out of service on this node, loudly, in the log. Without that check the
|
||||||
|
replica starts, passes its health checks, and hangs on its first all-reduce.
|
||||||
|
|
||||||
|
Fabric membership is configuration, not discovery, and there is no way around
|
||||||
|
that: OpenSM's default subnet prefix is the same on every fabric, so a matching
|
||||||
|
prefix proves nothing and a wrong guess produces a replica that starts, passes
|
||||||
|
health checks, and then hangs on its first all-reduce. You name the fabric when
|
||||||
|
you onboard the node, and two nodes are only in the same fabric if you said so.
|
||||||
|
|
||||||
|
### An engine
|
||||||
|
|
||||||
|
Pick what you will actually serve. For a first cluster, llama.cpp is the least
|
||||||
|
trouble and runs without a GPU.
|
||||||
|
|
||||||
|
```
|
||||||
|
# llama.cpp, from a release build
|
||||||
|
sudo apt-get install -y libgomp1 # it will not start without this
|
||||||
|
sudo mkdir -p /opt/llama.cpp
|
||||||
|
# unpack a llama.cpp release into /opt/llama.cpp
|
||||||
|
/opt/llama.cpp/llama-server --version
|
||||||
|
```
|
||||||
|
|
||||||
|
That `libgomp1` line is not padding. A `llama-server` binary missing it exists,
|
||||||
|
is executable, and cannot start — and the node reports `ready` throughout.
|
||||||
|
|
||||||
|
For vLLM, a virtualenv on the node:
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo apt-get install -y python3-venv
|
||||||
|
sudo python3 -m venv /opt/venvs/vllm
|
||||||
|
sudo /opt/venvs/vllm/bin/pip install vllm
|
||||||
|
/opt/venvs/vllm/bin/vllm --version
|
||||||
|
```
|
||||||
|
|
||||||
|
Check the engine still takes the flags llmgw builds for it, which drift between
|
||||||
|
releases:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw ctl engine-check vllm --bin /opt/venvs/vllm/bin/vllm
|
||||||
|
```
|
||||||
|
|
||||||
|
### Weights
|
||||||
|
|
||||||
|
Put them where the node can read them. `scp`, a shared filesystem, or the
|
||||||
|
engine's own downloader are all valid, and which you want depends on how many
|
||||||
|
nodes you have. If you would rather not choose, skip to the next section and
|
||||||
|
let llmgw fetch them.
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo install -d -o root -g root /var/lib/llmgw-models
|
||||||
|
# then place your .gguf or model directory there
|
||||||
|
```
|
||||||
|
|
||||||
|
For a first model, something small: a 3B at `q4_k_m` is about 2 GB, runs on
|
||||||
|
CPU, and answers fast enough to prove the whole path works.
|
||||||
|
|
||||||
|
### Letting llmgw fetch them
|
||||||
|
|
||||||
|
The simple arrangement, and the default: every node keeps its own copy. Point a
|
||||||
|
model's `source` at a file to download and the agent fetches it once, before the
|
||||||
|
engine is launched:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
source: hf:Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/qwen2.5-coder-3b-instruct-q4_k_m.gguf
|
||||||
|
# or an ordinary URL
|
||||||
|
source: https://models.internal/qwen2.5-coder-3b-q4_k_m.gguf
|
||||||
|
```
|
||||||
|
|
||||||
|
Eight shards of one replica starting together produce one download, not eight.
|
||||||
|
An interrupted fetch resumes rather than starting over, and a partial file is
|
||||||
|
never visible under the model's name — so a half-downloaded model cannot be
|
||||||
|
picked up by the next scan and handed to an engine.
|
||||||
|
|
||||||
|
Three kinds of source, and only one of them is fetched:
|
||||||
|
|
||||||
|
| `source` | What happens |
|
||||||
|
| --- | --- |
|
||||||
|
| `/var/lib/llmgw-models/x.gguf` | Checked, never copied. However it got there. |
|
||||||
|
| `hf:org/repo/file.gguf`, `https://…` | Fetched once into the cache, then reused |
|
||||||
|
| `Qwen/Qwen2.5-7B-Instruct` | Left alone — vLLM and SGLang download repositories themselves |
|
||||||
|
|
||||||
|
Which means the exotic arrangements need no configuration at all. Weights on a
|
||||||
|
Lustre mount, or an image exported over the fabric with SRP or NVMe-oF, are
|
||||||
|
simply *present*, and present means nothing is fetched.
|
||||||
|
|
||||||
|
### Telling llmgw where they are
|
||||||
|
|
||||||
|
Optional, and unset by default. Naming the directories in the agent's
|
||||||
|
`model_dirs` lets a node report which models it can see, and placement then
|
||||||
|
prefers a node that already holds one over a node that must first move tens of
|
||||||
|
gigabytes:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
model_dirs:
|
||||||
|
- /var/lib/llmgw-models
|
||||||
|
- /mnt/lustre/models
|
||||||
|
```
|
||||||
|
|
||||||
|
llmgw does not manage that storage and does not care how the files arrived. A
|
||||||
|
Lustre or NFS mount, an image exported over the fabric with SRP or NVMe-oF,
|
||||||
|
local NVMe, or an `scp` all look identical to it. A node with nothing configured
|
||||||
|
reports nothing and is treated exactly as every node was before this existed.
|
||||||
|
|
||||||
|
**It matters most when your fabrics are not connected to each other.** Weights
|
||||||
|
in `ib-a` are not weights in `ib-b`: the only path between them is the
|
||||||
|
management network, which is the slow one. `llmgw ctl status` shows what each
|
||||||
|
fabric holds, so a model seeded into one fabric does not quietly leave the rest
|
||||||
|
of the cluster unable to serve it.
|
||||||
|
|
||||||
|
How much disk a node needs follows from the models rather than a rule of thumb.
|
||||||
|
`llmgw ctl model plan <repo>` prints the sizes; as a starting point, a 70B at
|
||||||
|
bf16 is 141 GB and the same model at q4 is 43 GB.
|
||||||
|
|
||||||
|
## 5. Adding the node
|
||||||
|
|
||||||
|
Two halves, because they need different access: building the machine needs the
|
||||||
|
hypervisor, and issuing its identity needs the certificate authority. Where one
|
||||||
|
person holds both — the common case — one command does it:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw-add-node \
|
||||||
|
--name llmgw-node01 --ip 10.0.0.141/24 \
|
||||||
|
--server https://10.0.0.140:7443 \
|
||||||
|
--cores 16 --memory 131072 --disk 200 \
|
||||||
|
--fabric ib-a:mlx5_0,mlx5_1 \
|
||||||
|
--llamacpp-bin /opt/llama.cpp/llama-server
|
||||||
|
```
|
||||||
|
|
||||||
|
**The VM name, the hostname, the node id and the certificate's common name are
|
||||||
|
all the same string.** They do not have to be, and every time they are not,
|
||||||
|
somebody spends an afternoon working out which of the four a message refers to.
|
||||||
|
|
||||||
|
For a machine that already exists — bare metal, or a VM you built yourself — skip
|
||||||
|
provisioning and issue the identity only:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw-onboard-node \
|
||||||
|
--node llmgw-node01 --addr 10.0.0.141 \
|
||||||
|
--server https://10.0.0.140:7443 \
|
||||||
|
--fabric ib-a:mlx5_0,mlx5_1 \
|
||||||
|
--llamacpp-bin /opt/llama.cpp/llama-server
|
||||||
|
```
|
||||||
|
|
||||||
|
Onboarding needs SSH from the CA host to the node. It waits for SSH rather than
|
||||||
|
failing on it, and learns the host key of a machine it has not seen — a new
|
||||||
|
machine spends its first minute regenerating host keys and restarting sshd.
|
||||||
|
|
||||||
|
It also checks that a configured engine actually runs, and says so loudly when
|
||||||
|
it does not. That check exists because the failure is otherwise invisible.
|
||||||
|
|
||||||
|
Within a few seconds:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw ctl nodes
|
||||||
|
```
|
||||||
|
|
||||||
|
```
|
||||||
|
NODE STATE GPUS MEM CLASSES FABRICS REPLICAS LAST SEEN
|
||||||
|
llmgw-node01 ready 8/8 free 503G h100-80g ib-a 0 2s
|
||||||
|
```
|
||||||
|
|
||||||
|
If it does not appear, the control plane's log says why. A certificate whose
|
||||||
|
common name does not match the claimed node id is refused by design — that check
|
||||||
|
is what makes a node's identity something it proves rather than something it
|
||||||
|
picks.
|
||||||
|
|
||||||
|
Repeat for each node. Nodes in the same fabric must be given the same `--fabric`
|
||||||
|
name; nodes with no InfiniBand take no `--fabric` at all and are each their own
|
||||||
|
island, which is fine for single-node models.
|
||||||
|
|
||||||
|
## 6. Setting the node's CPU model, if it is a VM
|
||||||
|
|
||||||
|
Proxmox defaults a VM to `kvm64`, which hides AVX, AVX2, FMA and even SSE4.2
|
||||||
|
from the guest. Inference then falls back to scalar code and is not slightly
|
||||||
|
slower but unusable. Measured on one 8-core VM, same weights, same threads, the
|
||||||
|
only change being `--cpu host`:
|
||||||
|
|
||||||
|
| Qwen2.5-Coder | kvm64 | host |
|
||||||
|
| --- | ---: | ---: |
|
||||||
|
| 1.5B q4_K_M | 5.80 | **39.09** tok/s |
|
||||||
|
| 3B q4_K_M | 2.90 | **20.51** |
|
||||||
|
| 7B q4_K_M | 1.33 | **9.88** |
|
||||||
|
|
||||||
|
`llmgw-provision-node` sets `--cpu host` for you. A VM you built yourself
|
||||||
|
probably did not, and nothing about it looks broken — every number is simply
|
||||||
|
small. Check with `grep -qw avx2 /proc/cpuinfo` on the node.
|
||||||
|
|
||||||
|
## 7. Adding a model
|
||||||
|
|
||||||
|
Write the definition. This is the same shape a model takes in `server.yaml`, so
|
||||||
|
an entry moves between the two by being copied:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# qwen-coder-3b.yaml
|
||||||
|
name: qwen2.5-coder-3b-instruct
|
||||||
|
aliases: [coder]
|
||||||
|
source: /var/lib/llmgw-models/qwen2.5-coder-3b-instruct-q4_k_m.gguf
|
||||||
|
routing_profile: chat
|
||||||
|
# What a client may use in one request. Worth stating: llama.cpp divides
|
||||||
|
# --ctx-size across its parallel slots, so a pool of 65536 over two slots is
|
||||||
|
# 32768 each, and nothing else will tell you that.
|
||||||
|
context_length: 32768
|
||||||
|
max_output: 4096
|
||||||
|
scaling: { min: 1, max: 1 }
|
||||||
|
profiles:
|
||||||
|
- name: cpu
|
||||||
|
engine: llamacpp
|
||||||
|
weight: 1
|
||||||
|
priority: 100
|
||||||
|
startup_timeout: 5m
|
||||||
|
min_memory_gb: 8
|
||||||
|
args: ["--ctx-size", "65536"]
|
||||||
|
```
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw ctl model add -f qwen-coder-3b.yaml
|
||||||
|
llmgw ctl models
|
||||||
|
```
|
||||||
|
|
||||||
|
```
|
||||||
|
MODEL SERVING REPLICAS STATUS ALIASES
|
||||||
|
qwen2.5-coder-3b-instruct yes 1/1 ready coder
|
||||||
|
```
|
||||||
|
|
||||||
|
No restart, and it survives one: models added this way are stored, while
|
||||||
|
anything in `server.yaml` is declarative and returns on every start.
|
||||||
|
|
||||||
|
### Models that load when someone asks
|
||||||
|
|
||||||
|
A cluster with more models than GPUs can hold every model in its catalog and
|
||||||
|
only the wanted ones in its cards:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
scaling: { min: 0, max: 1, idle_timeout: 15m }
|
||||||
|
```
|
||||||
|
|
||||||
|
`min: 0` with a `max` above it means load on demand. The first request for the
|
||||||
|
model starts it and waits briefly; if it is not serving by then the client gets
|
||||||
|
a 503 with `Retry-After` and an honest "the model is loading", rather than being
|
||||||
|
told nothing serves it — a client told to retry succeeds where one told the
|
||||||
|
model does not exist gives up. Once nobody has asked for it in `idle_timeout`,
|
||||||
|
the replicas are stopped and the cards go back to the pool.
|
||||||
|
|
||||||
|
`min: 0, max: 0` is different and equally useful: configured, and deliberately
|
||||||
|
not running. A model with no `scaling` stanza at all gets that, because making
|
||||||
|
every unscaled model loadable on demand would let any client request start a
|
||||||
|
70B nobody intended to run.
|
||||||
|
|
||||||
|
Two things to weigh. A cold start is visible to whoever triggers it, so this
|
||||||
|
suits the long tail people are experimenting with rather than the two or three
|
||||||
|
models used daily — keep those at `min: 1`. And stopping a replica throws away
|
||||||
|
the KV cache prefix routing spent its effort filling, so an idle timeout wants
|
||||||
|
to be minutes rather than seconds.
|
||||||
|
|
||||||
|
When the cluster is full, a model somebody asks for will stop an idle one to
|
||||||
|
make room. Only idle replicas of other on-demand models: a model at `min: 1` is
|
||||||
|
a promise that it stays up, and breaking that to serve a different request would
|
||||||
|
make every guarantee in the catalog conditional on load. Only single-node
|
||||||
|
replicas, because evicting a multi-node one frees several machines and costs
|
||||||
|
tens of minutes to rebuild — a decision worth making deliberately. One at a
|
||||||
|
time. And never a replica that has just started, which is what stops two models
|
||||||
|
on a cluster with room for one from taking turns evicting each other.
|
||||||
|
|
||||||
|
For a GPU model, name the class and the width instead:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
profiles:
|
||||||
|
- name: tp2
|
||||||
|
engine: vllm
|
||||||
|
gpu_class: h100-80g # as `llmgw ctl nodes` reports it
|
||||||
|
tp: 2
|
||||||
|
priority: 100
|
||||||
|
```
|
||||||
|
|
||||||
|
If it does not place, ask:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw ctl explain-placement qwen2.5-coder-3b-instruct
|
||||||
|
```
|
||||||
|
|
||||||
|
which names every candidate and why each was rejected, rather than leaving you
|
||||||
|
to infer it.
|
||||||
|
|
||||||
|
## 8. The first request
|
||||||
|
|
||||||
|
```
|
||||||
|
curl http://<control-plane>:8080/v1/chat/completions \
|
||||||
|
-H "Authorization: Bearer $LLMGW_API_KEY" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"model": "coder", "messages": [{"role":"user","content":"hello"}]}'
|
||||||
|
```
|
||||||
|
|
||||||
|
Or open `/chat` in a browser and use it without a key at all — the page carries
|
||||||
|
your session, and the request still goes through admission, rate limits, budget
|
||||||
|
and accounting exactly as an API call does.
|
||||||
|
|
||||||
|
## Watching it
|
||||||
|
|
||||||
|
The control plane exports both halves on one endpoint: what the cluster is
|
||||||
|
serving — requests, latency, time to first token, tokens, admission, per
|
||||||
|
endpoint — and what it is made of, per node and per GPU, including memory,
|
||||||
|
temperature, ECC faults and utilisation.
|
||||||
|
|
||||||
|
**The dashboard is the answer for most clusters.** `/admin` shows capacity per
|
||||||
|
GPU class, free whole nodes per fabric, which replicas are holding hardware
|
||||||
|
nothing is asking for, who is using the cluster and what they are running, and
|
||||||
|
the same node and model tables `ctl` prints — with no monitoring stack to stand
|
||||||
|
up beside it. One binary, no dependencies, which is the point.
|
||||||
|
|
||||||
|
Prometheus is worth adding when you want history, alerts, or one view across
|
||||||
|
several clusters. The dashboard shows what is true now; it keeps nothing.
|
||||||
|
|
||||||
|
Metrics then get their own listener, because the admin one also carries pprof
|
||||||
|
and pprof can dump the process heap, where API keys sit in plaintext:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
admin_listen: "127.0.0.1:9090" # dashboard, management API, pprof — keep local
|
||||||
|
metrics_listen: "0.0.0.0:9091" # metrics only, safe to reach
|
||||||
|
```
|
||||||
|
|
||||||
|
One scrape target for the whole fleet. The node and GPU figures come from the
|
||||||
|
control plane, which already holds them, so there is nothing to install on the
|
||||||
|
nodes and no Prometheus configuration per machine.
|
||||||
|
|
||||||
|
A starting configuration, with the alerts worth having, is in
|
||||||
|
[deploy/monitoring/prometheus.yml](../deploy/monitoring/prometheus.yml).
|
||||||
|
|
||||||
|
Two figures repay knowing about. `llmgw_gpu_utilization_percent` is absent
|
||||||
|
rather than zero when the driver does not report it — a flat zero across a
|
||||||
|
fleet means the measurement is missing, not that the fleet is idle. And
|
||||||
|
`llmgw_gpu_nvlink_peers` at zero everywhere means the topology is unknown,
|
||||||
|
which is what a binary built without the nvml tag reports.
|
||||||
|
|
||||||
|
## Conversations across models
|
||||||
|
|
||||||
|
Off everywhere until a project asks. llmgw otherwise stores no message content
|
||||||
|
at all — usage rows carry token counts, costs and latencies, never text, and
|
||||||
|
even its own chat page keeps the conversation in the browser. A gateway that
|
||||||
|
keeps nothing anyone typed is a much easier thing to have in front of a team,
|
||||||
|
so keeping it is a decision someone makes rather than a default they discover.
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw keys threads --project default --retain-hours 24
|
||||||
|
```
|
||||||
|
|
||||||
|
The retention is required. A conversation store with no expiry is a transcript
|
||||||
|
of everything a team ever typed, kept forever because nobody chose a number.
|
||||||
|
The window is measured from the last turn, so a conversation still in progress
|
||||||
|
does not expire underneath it.
|
||||||
|
|
||||||
|
A client then sends a thread id and only its new turn:
|
||||||
|
|
||||||
|
```
|
||||||
|
curl $BASE/v1/chat/completions \
|
||||||
|
-H "Authorization: Bearer $LLMGW_API_KEY" \
|
||||||
|
-H "x-llmgw-thread: my-conversation" \
|
||||||
|
-d '{"model": "<model-name>", "messages": [{"role":"user","content":"and then?"}]}'
|
||||||
|
```
|
||||||
|
|
||||||
|
llmgw puts the stored history in front of that turn and sends it to whichever
|
||||||
|
model is serving — **including a different one from the one the conversation
|
||||||
|
started on**. That is the point: a thread begun on a large model can continue
|
||||||
|
on a small one, and a caller dropped to a cheaper model by a spent budget keeps
|
||||||
|
their conversation.
|
||||||
|
|
||||||
|
When the history does not fit the model now serving, the oldest turns are
|
||||||
|
dropped until it does, and the reply carries `x-llmgw-thread-trimmed` with how
|
||||||
|
many. System messages are never dropped, and neither is the turn the caller
|
||||||
|
just sent.
|
||||||
|
|
||||||
|
`DELETE /v1/threads/{id}` forgets one. It is a client endpoint on purpose: the
|
||||||
|
person who said the thing should be able to unsay it without asking an
|
||||||
|
operator to read it first. A thread belonging to another project answers the
|
||||||
|
same as one that never existed.
|
||||||
|
|
||||||
|
Turning it off again stops new storage, and existing conversations go at the
|
||||||
|
next hourly prune:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw keys threads --project default --off
|
||||||
|
```
|
||||||
|
|
||||||
|
## Memory notes for agents
|
||||||
|
|
||||||
|
Also off until a project asks, for the same reason. Where conversations are what
|
||||||
|
was said, notes are what an agent chose to keep, and they persist across
|
||||||
|
conversations:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw keys memory --project default --retain-hours 720
|
||||||
|
```
|
||||||
|
|
||||||
|
The retention is required and is measured from the last write to a note. An agent
|
||||||
|
then saves, searches and deletes notes through `/v1/memory` or, as tools, through
|
||||||
|
the MCP server at `/mcp` (`remember`, `recall`, `forget`). Both need the API key
|
||||||
|
of the project, and notes are private to the project and to the agent id sent in
|
||||||
|
`x-llmgw-agent`. A note in another project or agent answers the same as one that
|
||||||
|
does not exist. llmgw does not add these tools to requests itself.
|
||||||
|
|
||||||
|
Turning it off stops new storage, and existing notes go at the next hourly
|
||||||
|
prune:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw keys memory --project default --off
|
||||||
|
```
|
||||||
|
|
||||||
|
Without authentication, `threads.store` and `threads.retain_hours` in
|
||||||
|
`server.yaml` keep both conversations and notes in one shared pool. That is for
|
||||||
|
one trusted machine only; see [quickstart.md](quickstart.md).
|
||||||
|
|
||||||
|
## Reclaiming disk
|
||||||
|
|
||||||
|
`llmgw ctl nodes` has a DISK column: free space against the size it is free
|
||||||
|
out of, for the filesystems holding model weights. A dash means the node was
|
||||||
|
not asked — an agent with no `model_dirs`, or one older than this — and not
|
||||||
|
that it has no space. Shared filesystems are excluded from a node's figure,
|
||||||
|
because space on a filer belongs to everyone mounting it.
|
||||||
|
|
||||||
|
Weights llmgw downloaded for models that are no longer in the catalog can be
|
||||||
|
listed and removed:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw ctl weights # what could be freed, per node
|
||||||
|
llmgw ctl weights --collect # delete it
|
||||||
|
```
|
||||||
|
|
||||||
|
**Only files llmgw fetched itself are ever candidates.** They are recognisable
|
||||||
|
because llmgw names them after a hash of their source, so the set still wanted
|
||||||
|
is computed from the catalog exactly rather than guessed. Anything you put on
|
||||||
|
a node — by scp, by a shared mount, by hand — is left alone however
|
||||||
|
unreferenced it looks, and no llmgw command will remove it. The nodes check
|
||||||
|
every path again before deleting it, so a control plane that is wrong cannot
|
||||||
|
talk one into losing a model that took a week to copy.
|
||||||
|
|
||||||
|
Which also means llmgw will not reclaim the storage most likely to be filling
|
||||||
|
your disk. Weights you placed yourself are yours to remove.
|
||||||
|
|
||||||
|
A profile may declare `min_disk_gb`, and a node without that much free is then
|
||||||
|
not a candidate for it — checked only on nodes that do not already hold the
|
||||||
|
weights, since those need no room for them. Without the field, disk is not
|
||||||
|
considered, which is how every profile written before it keeps behaving.
|
||||||
|
|
||||||
|
## When something does not work
|
||||||
|
|
||||||
|
**The node is `ready` but every replica fails.** Almost always a missing engine
|
||||||
|
or weights. `llmgw ctl replicas` shows the shard's error and the engine's last
|
||||||
|
output, which is the actual diagnosis.
|
||||||
|
|
||||||
|
**A model will not place.** `llmgw ctl explain-placement <model>` names each
|
||||||
|
node and the reason: wrong GPU class, not enough free cards, too little memory,
|
||||||
|
no fabric, a label it does not carry.
|
||||||
|
|
||||||
|
**Everything is slow.** On a VM, check AVX2 (step 6). On CPU, check the
|
||||||
|
quantisation — the difference between `q8_0` and `q4_k_m` is roughly a factor of
|
||||||
|
two. `llmgw ctl status` shows how full the cluster is.
|
||||||
|
|
||||||
|
**Tool calls come back as text.** The model made the call and the engine did not
|
||||||
|
parse it. `llmgw ctl tool-check --endpoint http://<node>:<port>` distinguishes
|
||||||
|
that from a model ignoring its tools, which has a different fix.
|
||||||
|
|
||||||
|
**Requests fail at exactly sixty seconds.** `timeouts.response_header` is a
|
||||||
|
liveness check for streaming and a total generation budget for non-streaming
|
||||||
|
requests. On a slow model, set `timeouts.generation` higher.
|
||||||
|
|
||||||
|
## What to read next
|
||||||
|
|
||||||
|
- `docs/node-onboarding.md` — adding nodes in more detail, and what onboarding
|
||||||
|
deliberately does not do
|
||||||
|
- `docs/onboarding.md` — giving people accounts and keys
|
||||||
|
- `llmgw-plan.md` — why the system is shaped the way it is
|
||||||
|
|
||||||
|
## Known gaps, so you find them here rather than at 2am
|
||||||
|
|
||||||
|
- **A fetch is silent while it runs.** llmgw will fetch weights for you, but
|
||||||
|
it reports no progress, so a first start on a large model looks like a hang.
|
||||||
|
`ls -l` in the model directory is the only progress bar there is.
|
||||||
|
- **Every node fetches its own copy.** There is no seeding between nodes, so
|
||||||
|
bringing up eight nodes on a new model is eight downloads from the same
|
||||||
|
origin. On a fabric where the nodes can see each other this is wasteful, and
|
||||||
|
it is the obvious next thing to build.
|
||||||
|
- **Engines are yours to install.** Onboarding verifies one runs; it installs
|
||||||
|
nothing.
|
||||||
|
- **One control plane.** No HA yet; it is a single process with a SQLite file.
|
||||||
|
Back up `/var/lib/llmgw/llmgw.db`.
|
||||||
|
- **Multi-node replicas have not been run on real InfiniBand.** The scheduling,
|
||||||
|
the fabric invariant and the rendezvous are exercised against simulated
|
||||||
|
hardware only. Single-node placement on real GPUs is the well-trodden path;
|
||||||
|
if you are the first to run a TP group across real cards, expect to find
|
||||||
|
something.
|
||||||
@@ -0,0 +1,230 @@
|
|||||||
|
# Adding a node
|
||||||
|
|
||||||
|
Written for whoever runs the cluster.
|
||||||
|
|
||||||
|
## The short version
|
||||||
|
|
||||||
|
Where one person holds both the hypervisor and the certificate authority —
|
||||||
|
which is the common case — one command does the whole thing in about
|
||||||
|
45 seconds:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw-add-node \
|
||||||
|
--name llmgw-node05 --ip 10.0.0.145/24 \
|
||||||
|
--server https://10.0.0.140:7443 \
|
||||||
|
--cores 8 --memory 32768 --disk 60 \
|
||||||
|
--llamacpp-bin /opt/llama.cpp/llama-server
|
||||||
|
```
|
||||||
|
|
||||||
|
The VMID defaults to the last octet of the address, so `qm list` and
|
||||||
|
`llmgw ctl nodes` stay in the same order. `--provision-only` and
|
||||||
|
`--onboard-only` split it back into the two halves below when the two kinds
|
||||||
|
of access belong to different people.
|
||||||
|
|
||||||
|
The rest of this page is what that command does, and what it still does not.
|
||||||
|
|
||||||
|
## The two halves
|
||||||
|
|
||||||
|
Two scripts, because the two halves of onboarding are done by different people
|
||||||
|
with different access: one needs the hypervisor, the other needs the
|
||||||
|
certificate authority.
|
||||||
|
|
||||||
|
## 1. The machine
|
||||||
|
|
||||||
|
On the Proxmox host:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw-provision-node --vmid 144 --name llmgw-node04 --ip 10.0.0.144/24
|
||||||
|
```
|
||||||
|
|
||||||
|
Name it after the node id. **The VM name, the hostname, the node id and the
|
||||||
|
certificate's common name are all the same string.** They do not have to be, and
|
||||||
|
every time they are not, somebody spends an afternoon working out which of the
|
||||||
|
four a message is talking about.
|
||||||
|
|
||||||
|
It imports a cloud image and lets cloud-init do the rest, which is why a node
|
||||||
|
takes a minute rather than an afternoon. It refuses to touch an existing VMID:
|
||||||
|
adopting a machine is not a decision a provisioning script should make.
|
||||||
|
|
||||||
|
Releases carry static `linux/amd64` and `linux/arm64` binaries. Download one and
|
||||||
|
pass it with `--binary` rather than copying whatever happens to be on the
|
||||||
|
machine you are running onboarding from — otherwise whoever runs the script
|
||||||
|
quietly becomes the build server, and the fleet ends up running whatever was in
|
||||||
|
their working tree.
|
||||||
|
|
||||||
|
## 2. The identity
|
||||||
|
|
||||||
|
On the host that holds the certificate authority — this is the part that cannot
|
||||||
|
be delegated, because it issues the credential by which the control plane
|
||||||
|
believes the node is who it says it is:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw-onboard-node \
|
||||||
|
--node llmgw-node04 \
|
||||||
|
--addr 10.0.0.144 \
|
||||||
|
--server https://10.0.0.140:7443 \
|
||||||
|
--fabric ib-b
|
||||||
|
```
|
||||||
|
|
||||||
|
It issues a client certificate whose common name is the node id, installs it
|
||||||
|
along with the CA root, trusts that root system-wide so other tooling on the
|
||||||
|
machine stops treating llmgw's certificates as strangers, installs the binary
|
||||||
|
and the unit, and starts the agent. The node appears in `llmgw ctl nodes` within
|
||||||
|
a few seconds.
|
||||||
|
|
||||||
|
If it does not, the control plane's log says why. A certificate whose common
|
||||||
|
name does not match the claimed node id is refused by design — that check is
|
||||||
|
what makes a node's identity something it proves rather than something it picks.
|
||||||
|
|
||||||
|
**It needs ssh from the CA host to the node.** Provisioning copies that host's
|
||||||
|
`authorized_keys` into the new machine, so a node built by the script above
|
||||||
|
already satisfies it; an older machine may need the key added.
|
||||||
|
|
||||||
|
It waits for ssh rather than failing on it, and learns the host key of a machine
|
||||||
|
it has not seen (`accept-new`, which still refuses a *changed* key — the risk
|
||||||
|
being guarded against is a substituted machine, not a new one). Both are needed
|
||||||
|
to run the halves back to back: for the first minute of its life a new machine
|
||||||
|
is still regenerating host keys and restarting sshd, so it refuses connections
|
||||||
|
it accepted a moment earlier.
|
||||||
|
|
||||||
|
It also checks that a configured engine actually runs, which is cheap here and
|
||||||
|
expensive later — see below.
|
||||||
|
|
||||||
|
## The CPU model matters, and its default is wrong
|
||||||
|
|
||||||
|
Nodes are provisioned with `--cpu host`. Proxmox's default, `kvm64`, hides
|
||||||
|
AVX, AVX2, FMA and even SSE4.2 from the guest, and llama.cpp then loads its
|
||||||
|
scalar backend instead of the one built for the host's instruction set.
|
||||||
|
Measured on node05, an 8-core VM on a Xeon E5-2680 v4, same weights, same
|
||||||
|
threads, the only change being the CPU model:
|
||||||
|
|
||||||
|
| Qwen2.5-Coder | kvm64 tok/s | host tok/s |
|
||||||
|
| -------------- | ----------: | ---------: |
|
||||||
|
| 1.5B q4_K_M | 5.80 | 39.09 |
|
||||||
|
| 3B q4_K_M | 2.90 | 20.51 |
|
||||||
|
| 7B q4_K_M | 1.33 | 9.88 |
|
||||||
|
|
||||||
|
Prompt processing is worse still: 7.6 tokens a second against 143.8, a factor
|
||||||
|
of nineteen.
|
||||||
|
|
||||||
|
It is worth calling out because nothing looks broken. The node registers, the
|
||||||
|
engine starts, replies are correct — every number is simply small, and the
|
||||||
|
obvious conclusion is that CPU inference is hopeless rather than that the VM
|
||||||
|
cannot multiply efficiently.
|
||||||
|
|
||||||
|
The trade is live migration: a VM pinned to the host CPU cannot move to a
|
||||||
|
machine with a different one. For a node whose job is arithmetic that is the
|
||||||
|
right way round, and `--cpu` overrides it where it is not.
|
||||||
|
|
||||||
|
## What onboarding does not do
|
||||||
|
|
||||||
|
It gives a machine an identity and an agent. It does **not** install inference
|
||||||
|
engines or model weights, and a node missing either looks healthy in
|
||||||
|
`llmgw ctl nodes` right up until a replica is placed on it and the shard dies
|
||||||
|
with "executable file not found".
|
||||||
|
|
||||||
|
Onboarding does check that an engine you point it at can be executed, and says
|
||||||
|
so loudly when it cannot. That check exists because the failure is invisible
|
||||||
|
otherwise: `llama-server` on node05 was present, executable, and could not start
|
||||||
|
for want of `libgomp1`, and the node reported `ready` in `llmgw ctl nodes` the
|
||||||
|
whole time. Being told at onboarding beats finding out when a replica is placed
|
||||||
|
and the reason is three logs away.
|
||||||
|
|
||||||
|
So the node is not finished until:
|
||||||
|
|
||||||
|
- the engines its catalog profiles name are installed (`vllm`, `sglang`,
|
||||||
|
`llama-server`), at the paths the agent configuration gives
|
||||||
|
- the weights are on disk, at the `source` each model declares
|
||||||
|
|
||||||
|
Both are deliberately outside this script. Engines are a platform decision with
|
||||||
|
versions and drivers attached, and distributing weights is its own problem —
|
||||||
|
hundreds of gigabytes that want a shared filesystem or a cache, not an scp from
|
||||||
|
a laptop. llmgw will fetch weights for a model whose `source` is a URL, one
|
||||||
|
copy per node, which covers the simple case; anything larger stays the
|
||||||
|
operator's arrangement, and llmgw is content to find the files already there.
|
||||||
|
|
||||||
|
The `sim` engine needs neither, which is what makes it useful for exercising
|
||||||
|
everything else.
|
||||||
|
|
||||||
|
## Checking an engine after an upgrade
|
||||||
|
|
||||||
|
Engines rename and remove flags between versions, and when that happens the
|
||||||
|
failure arrives minutes into a model load, in the engine's own error message, on
|
||||||
|
a node nobody was watching.
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw ctl engine-check vllm
|
||||||
|
llmgw ctl engine-check llamacpp --help-output "$(llama-server --help 2>&1)"
|
||||||
|
```
|
||||||
|
|
||||||
|
It builds the widest command llmgw would ever produce — a multi-node one, since
|
||||||
|
a flag that only appears there is exactly the one nobody notices has gone — and
|
||||||
|
checks every flag against the engine's own `--help`. No GPU required: a CLI
|
||||||
|
prints its help on any machine.
|
||||||
|
|
||||||
|
Some programs do not give up their whole help for `--help`. vLLM prints a
|
||||||
|
summary of section names and expects `--help=all`; checking against the summary
|
||||||
|
reports every flag as missing, confidently and wrongly. `engine-check` knows
|
||||||
|
that one. If you add an engine, check that what you get back is the full list.
|
||||||
|
|
||||||
|
vLLM also refuses to do anything at all on a machine with no accelerator,
|
||||||
|
including print its help, unless `VLLM_TARGET_DEVICE` says what to assume.
|
||||||
|
`engine-check` sets it.
|
||||||
|
|
||||||
|
Give it the **whole** help. Truncating llama.cpp's 725 lines to 400 reports five
|
||||||
|
flags as missing that the running cluster uses; the command warns when the text
|
||||||
|
it was given does not document `--help` itself, which is the cheapest sign it
|
||||||
|
was cut short.
|
||||||
|
|
||||||
|
## Fabrics
|
||||||
|
|
||||||
|
`--fabric` is how a node says which InfiniBand fabric it is attached to, and
|
||||||
|
there is no way around stating it. OpenSM's default subnet prefix is `fe80::` on
|
||||||
|
every fabric, so a matching prefix proves nothing and a wrong guess produces a
|
||||||
|
replica that starts, passes health checks, and then hangs on its first
|
||||||
|
all-reduce.
|
||||||
|
|
||||||
|
Omit it for a node with no fabric. Such a node is its own island: it can host
|
||||||
|
single-node replicas and will never be given a multi-node one.
|
||||||
|
|
||||||
|
Two adapters per fabric is the default; `--fabric ib-b:mlx5_2,mlx5_3` names them
|
||||||
|
explicitly. An adapter must never appear under two fabrics — the agent refuses
|
||||||
|
that configuration, because the consequence is the hang above.
|
||||||
|
|
||||||
|
## Where the driver is
|
||||||
|
|
||||||
|
```
|
||||||
|
--nvidia-smi /opt/nvidia/bin/nvidia-smi
|
||||||
|
--ib-root /sys/class/infiniband
|
||||||
|
```
|
||||||
|
|
||||||
|
Both default to the usual places. Containerized driver installs routinely put
|
||||||
|
`nvidia-smi` somewhere else, and an agent that cannot find it registers with no
|
||||||
|
GPUs — which looks exactly like a node that has none.
|
||||||
|
|
||||||
|
## Removing a node
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw ctl drain llmgw-node04 # move its replicas off, one at a time
|
||||||
|
llmgw ctl nodes # wait until it holds none
|
||||||
|
```
|
||||||
|
|
||||||
|
Then stop the agent. Draining brings each replacement up before stopping the
|
||||||
|
original, which is the difference between a rolling move and an outage.
|
||||||
|
|
||||||
|
The node's certificate stays valid until it expires. There is no revocation list
|
||||||
|
— if a machine is decommissioned rather than retired, reissue the authority or
|
||||||
|
accept that a certificate naming a node that no longer exists can still connect
|
||||||
|
and claim to be it.
|
||||||
|
|
||||||
|
## Testing without hardware
|
||||||
|
|
||||||
|
Every part of this works on machines with no GPUs and no InfiniBand. Point
|
||||||
|
`--nvidia-smi` at a script that prints the CSV discovery asks for, point
|
||||||
|
`--ib-root` at a sysfs-shaped directory, and use `engine: sim` in the catalog.
|
||||||
|
|
||||||
|
The simulated engine rendezvouses for real: rank 0 binds the dist-init port and
|
||||||
|
does not serve until every worker has joined, and workers serve nothing. That
|
||||||
|
exercises the multi-node launch sequence, the fabric constraint, and failure and
|
||||||
|
rebuild — which is most of what goes wrong — without exercising NCCL, which is
|
||||||
|
the rest.
|
||||||
@@ -0,0 +1,319 @@
|
|||||||
|
# Onboarding a user
|
||||||
|
|
||||||
|
Written for whoever operates an llmgw deployment: the person with a shell on the
|
||||||
|
control plane machine.
|
||||||
|
|
||||||
|
There are two ways a person gets a key. Pick one and put it in the config.
|
||||||
|
|
||||||
|
| | Self-service | Operator-issued |
|
||||||
|
|---|---|---|
|
||||||
|
| Config | `auth.signup_open: true` | `auth.signup_open: false` |
|
||||||
|
| They do | sign up, wait, then make their own keys | wait for a link |
|
||||||
|
| You do | approve once | issue every key |
|
||||||
|
| Good for | more than a handful of people | a few, or machines |
|
||||||
|
|
||||||
|
Self-service does not mean open. A signup produces an account that can do
|
||||||
|
nothing at all; **approval is what makes it work**, and approval needs a shell
|
||||||
|
here rather than a session cookie, because it is the step that decides who gets
|
||||||
|
to spend the cluster.
|
||||||
|
|
||||||
|
Everything below runs as the user the server runs as, so the files it creates
|
||||||
|
stay readable by it:
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo -u llmgw llmgw <command> --store /var/lib/llmgw/llmgw.db
|
||||||
|
```
|
||||||
|
|
||||||
|
## Self-service
|
||||||
|
|
||||||
|
Turn on the form:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
auth:
|
||||||
|
enabled: true
|
||||||
|
store: /var/lib/llmgw/llmgw.db
|
||||||
|
signup_open: true
|
||||||
|
```
|
||||||
|
|
||||||
|
The landing page then links to it, and the rest is:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw users list --pending --store …
|
||||||
|
ID EMAIL NAME STATE PROJECT SIGNED UP
|
||||||
|
usr-k4mq… ayse@example.com Ayşe pending - 4m12s ago
|
||||||
|
|
||||||
|
llmgw users approve usr-k4mq… --project default --by ilker --store …
|
||||||
|
```
|
||||||
|
|
||||||
|
That is the whole operator loop. From then on they sign in at `/login`, and
|
||||||
|
`/profile` is where they create keys, see when each was last used, and revoke the
|
||||||
|
ones they are done with. A key is shown once on the page that creates it and
|
||||||
|
never again.
|
||||||
|
|
||||||
|
**This gateway sends no email.** There is no "your account was approved" message
|
||||||
|
and no password reset, so tell people what to expect: they sign up, they tell
|
||||||
|
you, you approve, they sign in. If that loop is too manual for your size, it is
|
||||||
|
the thing to automate — not the approval step.
|
||||||
|
|
||||||
|
To close an account:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw users suspend usr-k4mq… --store …
|
||||||
|
```
|
||||||
|
|
||||||
|
which closes their sessions immediately. Their **API keys keep working** — those
|
||||||
|
are separate credentials, and revoking them is a separate step:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw keys list --project default --store …
|
||||||
|
llmgw keys revoke key-… --store …
|
||||||
|
```
|
||||||
|
|
||||||
|
That separation is deliberate: a suspended person's running jobs should not all
|
||||||
|
fail at once without someone deciding they should.
|
||||||
|
|
||||||
|
## Operator-issued
|
||||||
|
|
||||||
|
With `signup_open: false` there is no form, and you issue keys yourself. This is
|
||||||
|
also the path for things that are not people — CI, a notebook server, a script.
|
||||||
|
|
||||||
|
### Decide where the key belongs
|
||||||
|
|
||||||
|
Keys hang off a project, projects off a team, teams off an organization. Limits
|
||||||
|
can be set at any of those levels and compose downward by taking the **most
|
||||||
|
restrictive** value — a key cannot raise what its team set, and a team cannot
|
||||||
|
raise what its organization set. That is the only rule under which a budget
|
||||||
|
means anything.
|
||||||
|
|
||||||
|
If you have no structure yet, one of each is fine:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw keys init --store … --org mynodes --team default --project default
|
||||||
|
```
|
||||||
|
|
||||||
|
It is idempotent: running it again renames rather than duplicating, so it can
|
||||||
|
live in whatever provisions the machine.
|
||||||
|
|
||||||
|
Split it when you want separate limits or separate reporting, not before.
|
||||||
|
|
||||||
|
### Issue the key as a one-time link
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw keys create --store … \
|
||||||
|
--project default \
|
||||||
|
--band gold \
|
||||||
|
--base-url https://llmgw.example \
|
||||||
|
--claim \
|
||||||
|
"ayse-laptop"
|
||||||
|
|
||||||
|
https://llmgw.example/claim/3mKq…
|
||||||
|
```
|
||||||
|
|
||||||
|
Send the link, not the key.
|
||||||
|
|
||||||
|
A key pasted into a chat window stays in that chat window forever, searchable by
|
||||||
|
everyone who joins the channel later. A link works once and then expires, so the
|
||||||
|
worst case is someone opening it before the intended person — which you find out
|
||||||
|
about, because they will tell you the link is dead.
|
||||||
|
|
||||||
|
The name is for you, not the user: it appears in `keys list` and is how you will
|
||||||
|
know which key to revoke.
|
||||||
|
|
||||||
|
**Why the link is safe to store.** The key is sealed with the link's own token,
|
||||||
|
and the token is never written down — only its hash is. A database leak yields
|
||||||
|
two useless hashes and a blob nothing in the database can open.
|
||||||
|
|
||||||
|
Without `--claim` the key is printed to stdout instead, for when you are
|
||||||
|
provisioning something that is not a person:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw keys create --store … --project ci "github-runner" > /run/secrets/llmgw-key
|
||||||
|
```
|
||||||
|
|
||||||
|
## Limits
|
||||||
|
|
||||||
|
Limits are optional. A key with none is unlimited, which is the right default on
|
||||||
|
a private cluster and the wrong one the moment the gateway faces anything else.
|
||||||
|
|
||||||
|
```
|
||||||
|
# Everyone in this project
|
||||||
|
llmgw keys limits --store … --scope project --id default \
|
||||||
|
--rpm 60 --tpm 200000
|
||||||
|
|
||||||
|
# One key, tighter
|
||||||
|
llmgw keys limits --store … --scope key --id key-mpmip77twpt7qaxc \
|
||||||
|
--rpm 10
|
||||||
|
|
||||||
|
# Restrict which models
|
||||||
|
llmgw keys limits --store … --scope project --id default \
|
||||||
|
--models qwen-0.5b,llama-3.3-70b
|
||||||
|
```
|
||||||
|
|
||||||
|
An **absent** model list permits everything. An **empty** one permits nothing —
|
||||||
|
`--models ""` is a way to suspend a project without revoking its keys.
|
||||||
|
|
||||||
|
Allowlists compose by intersection: a key cannot add a model its team withheld.
|
||||||
|
A model's canonical name and any of its aliases name the same thing, so either
|
||||||
|
form works.
|
||||||
|
|
||||||
|
Budgets are counted in micro-units, and what a unit means is yours to decide.
|
||||||
|
Self-hosted models have no per-token price, so it can be GPU-seconds, a notional
|
||||||
|
rate, or 1 000 000 everywhere — which turns a budget into a plain token quota.
|
||||||
|
Models are priced in the server's `auth.pricing` section; a model priced at zero
|
||||||
|
is exempt from budget checks, which is what lets a key with nothing left still be
|
||||||
|
served a cheap fallback.
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw keys limits --store … --scope project --id default \
|
||||||
|
--budget 50000000 --period month # 50 units a month
|
||||||
|
```
|
||||||
|
|
||||||
|
## The landing page
|
||||||
|
|
||||||
|
Someone who reaches the gateway without a key sees the landing page. Tell it who
|
||||||
|
to send them to, or it can only shrug:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
landing:
|
||||||
|
contact: "platform@example.com" # or an https:// URL, or "#platform on Slack"
|
||||||
|
notice: "Internal use only."
|
||||||
|
```
|
||||||
|
|
||||||
|
An address becomes a mailto link, an `http(s)` URL stays a link, anything else is
|
||||||
|
shown as text. Other schemes are never linked — configuration is not always
|
||||||
|
written by whoever runs the gateway, and a `javascript:` href on a page served to
|
||||||
|
every visitor is a short path to a stored XSS.
|
||||||
|
|
||||||
|
## What the user sees
|
||||||
|
|
||||||
|
They open the link, press the button, and get the key once. The page shows them
|
||||||
|
the base URL and a working `curl`, and points at two endpoints:
|
||||||
|
|
||||||
|
- `GET /v1/models` — what this key may use
|
||||||
|
- `GET /me` — its limits and what it has spent
|
||||||
|
|
||||||
|
`/me` is there so nobody has to ask you "why was I rate limited" or "how much
|
||||||
|
have I used". It is outside `/v1` because it is not part of the OpenAI API.
|
||||||
|
|
||||||
|
## Rotation and revocation
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw keys list --store … --project default
|
||||||
|
```
|
||||||
|
|
||||||
|
`LAST USED` is written in batches rather than per request, so it lags by up to a
|
||||||
|
minute. It is the column that tells you whether a key is still in use, which is
|
||||||
|
what makes rotation safe:
|
||||||
|
|
||||||
|
**To rotate**, issue the new key, let both work while the user switches, watch
|
||||||
|
`LAST USED` on the old one stop moving, then revoke it.
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw keys revoke --store … key-mpmip77twpt7qaxc
|
||||||
|
```
|
||||||
|
|
||||||
|
Revocation is a soft delete: the key stops working, the row stays, and its usage
|
||||||
|
history keeps its subject. A running server keeps accepting a revoked key until
|
||||||
|
its cache expires — `auth.cache_ttl`, 30 seconds by default. That is the price of
|
||||||
|
keeping the database off the request path, and it is the number to lower if you
|
||||||
|
need revocation to bite faster.
|
||||||
|
|
||||||
|
## Passwords
|
||||||
|
|
||||||
|
People change their own from `/profile`. It asks for the current one — a session
|
||||||
|
is what someone gets from a borrowed laptop, and the password is what they do not
|
||||||
|
have — and it signs every other browser out, which is the point if they are
|
||||||
|
changing it because something went wrong.
|
||||||
|
|
||||||
|
There is no self-service reset, because this gateway sends no email and a reset
|
||||||
|
link with nowhere to send it is not a reset. When someone loses theirs:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw users reset-password usr-k4mq… --password "a temporary passphrase" --store …
|
||||||
|
```
|
||||||
|
|
||||||
|
Tell them out of band and have them change it from their profile. Their sessions
|
||||||
|
are closed — if they lost the password, the reason might be that someone else has
|
||||||
|
it. Their API keys keep working; those are separate credentials.
|
||||||
|
|
||||||
|
## Trying a model in the browser
|
||||||
|
|
||||||
|
`/chat` is a playground for anyone signed in. It exists so somebody can see
|
||||||
|
whether a model works without setting up a client, and — more often — so they
|
||||||
|
can tell whether a problem is the gateway or their code.
|
||||||
|
|
||||||
|
**The browser never holds a key.** The page carries a session; the server turns
|
||||||
|
that into the same identity an API key would produce, resolved from the person's
|
||||||
|
project, and the request goes down the ordinary path: admission, model
|
||||||
|
allowlist, rate limits, budget, accounting. Anything else would make the page a
|
||||||
|
hole in every limit the rest of the gateway enforces.
|
||||||
|
|
||||||
|
Usage from the page is attributed to `session:<user-id>` rather than a key, so
|
||||||
|
it is billed to the right project and visibly did not come from a credential
|
||||||
|
somebody holds. One turn is capped at 512 tokens: a playground should not be the
|
||||||
|
thing that ties up a replica for ten minutes, and anyone who wants a longer
|
||||||
|
answer has an API key for that.
|
||||||
|
|
||||||
|
Nothing is stored. The conversation lives in the page and is gone when it is
|
||||||
|
closed.
|
||||||
|
|
||||||
|
## Seeing the cluster
|
||||||
|
|
||||||
|
The dashboard — nodes, replicas, models, why something did not get placed, usage
|
||||||
|
per key — is served twice, and the difference matters.
|
||||||
|
|
||||||
|
On the **admin listener** (`127.0.0.1:9090` by default) it needs no
|
||||||
|
authentication: having a shell on the machine is the credential.
|
||||||
|
|
||||||
|
```
|
||||||
|
ssh -L 9090:127.0.0.1:9090 <control-plane> # then http://127.0.0.1:9090/
|
||||||
|
```
|
||||||
|
|
||||||
|
That listener also serves pprof, which can dump the process heap — where API
|
||||||
|
keys sit in plaintext. **Never expose that port.** It is bound to loopback for
|
||||||
|
that reason.
|
||||||
|
|
||||||
|
On the **client-facing port** the same page is at `/admin`, behind a session and
|
||||||
|
an admin flag, so it can be reached from a browser without the port carrying
|
||||||
|
pprof along with it:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw users promote usr-k4mq… --store …
|
||||||
|
```
|
||||||
|
|
||||||
|
Someone without the flag gets a 404 rather than a 403, because 403 confirms the
|
||||||
|
page exists.
|
||||||
|
|
||||||
|
## Managing accounts from the browser
|
||||||
|
|
||||||
|
`/admin/users` lists who is waiting and who is active. Approving (into a project)
|
||||||
|
and suspending are there, because that is the work an operator does constantly
|
||||||
|
and it should not need a terminal.
|
||||||
|
|
||||||
|
Three things are deliberately not there:
|
||||||
|
|
||||||
|
| | Why |
|
||||||
|
|---|---|
|
||||||
|
| `llmgw users promote` | An admin who can make admins turns a stolen session into permanent access |
|
||||||
|
| `llmgw users reset-password` | Setting someone's password is taking over their account |
|
||||||
|
| `llmgw keys create` | Issuing a credential on another person's behalf |
|
||||||
|
|
||||||
|
Each of those turns a stolen admin cookie into more than it was, and each is
|
||||||
|
rare enough that a shell is no hardship. Cordon, drain and everything else that
|
||||||
|
touches the cluster are also CLI-only for the same reason: the dashboard shows
|
||||||
|
the cluster, it does not steer it.
|
||||||
|
|
||||||
|
Every administrative action is logged with who did it, including the ones that
|
||||||
|
were refused.
|
||||||
|
|
||||||
|
## What this does not do
|
||||||
|
|
||||||
|
- **No email.** No approval notice, no self-service password reset, no address
|
||||||
|
verification. The loop is manual on purpose at this size; if it stops being
|
||||||
|
the right size, that is the thing to automate.
|
||||||
|
- **No SSO.** Accounts are local, with local passwords.
|
||||||
|
- **No roles.** There is one flag, `is_admin`, and it grants looking at the
|
||||||
|
dashboard. Everything else an administrator does is the CLI, and having a
|
||||||
|
shell here is what makes someone one.
|
||||||
|
- **Rate limits are per gateway instance.** With two instances each counts its own,
|
||||||
|
so the effective limit doubles. v1 is single-instance by design.
|
||||||
@@ -0,0 +1,375 @@
|
|||||||
|
# Quickstart: one machine, one model, ten minutes
|
||||||
|
|
||||||
|
Written for someone who wants to see llmgw answer a request on a single
|
||||||
|
workstation before reading anything else. No systemd, no `sudo`. For a real
|
||||||
|
cluster, see [install.md](install.md).
|
||||||
|
|
||||||
|
What you end up with: an OpenAI-compatible endpoint on `localhost:8080` in front
|
||||||
|
of a real model served by llama.cpp.
|
||||||
|
|
||||||
|
## Two ways to run
|
||||||
|
|
||||||
|
Steps 1 to 4 use the simple one, **router-only**: you start the engines yourself
|
||||||
|
and tell the router where they are with `--endpoint`. No certificates, no node
|
||||||
|
agent.
|
||||||
|
|
||||||
|
Step 5 uses the **control plane and node agent**, the same architecture as a real
|
||||||
|
cluster on one machine: the agent starts and supervises the engine, and the
|
||||||
|
control plane places the model.
|
||||||
|
|
||||||
|
## 1. Build
|
||||||
|
|
||||||
|
```
|
||||||
|
git clone https://gitea.mynodes.xyz/ilker/llmgw && cd llmgw
|
||||||
|
make build
|
||||||
|
make test
|
||||||
|
./bin/llmgw --version
|
||||||
|
```
|
||||||
|
|
||||||
|
`make build` writes `./bin/llmgw`. Go 1.26 or later is the only requirement.
|
||||||
|
|
||||||
|
## 2. Check the path with a fake engine
|
||||||
|
|
||||||
|
Before involving a GPU, prove the router works. `sim-engine` is an
|
||||||
|
OpenAI-compatible fake that streams placeholder words.
|
||||||
|
|
||||||
|
```
|
||||||
|
./bin/llmgw sim-engine --listen :8001 --model llama &
|
||||||
|
./bin/llmgw server --endpoint e1=http://127.0.0.1:8001=llama &
|
||||||
|
|
||||||
|
curl -N localhost:8080/v1/chat/completions \
|
||||||
|
-H 'content-type: application/json' \
|
||||||
|
-H 'x-session-id: demo' \
|
||||||
|
-d '{"model":"llama","messages":[{"role":"user","content":"hello"}],"stream":true}'
|
||||||
|
```
|
||||||
|
|
||||||
|
Chunks of `alpha beta gamma ...` ending in `data: [DONE]` mean the router is
|
||||||
|
fine. Stop both before continuing (see [Stopping](#stopping)).
|
||||||
|
|
||||||
|
## 3. Serve a real model
|
||||||
|
|
||||||
|
Two processes, two ports. Clients only ever talk to the router; the router
|
||||||
|
knows where the engine is because you tell it with `--endpoint`:
|
||||||
|
|
||||||
|
```
|
||||||
|
curl :8080 -> llmgw router :8080 -> llama serve :8082 -> model
|
||||||
|
(client) (the gateway) (the engine)
|
||||||
|
```
|
||||||
|
|
||||||
|
Requests go to 8080. Port 8082 is the engine's own, and you only touch it to
|
||||||
|
check that the engine is up.
|
||||||
|
|
||||||
|
llmgw does not install engines or weights. You need:
|
||||||
|
|
||||||
|
- a `llama-server` build from llama.cpp (CUDA build if you want the GPU), and
|
||||||
|
- a `.gguf` file.
|
||||||
|
|
||||||
|
Check the card first with `nvidia-smi`. An 8 GB card holds a 1B model at bf16
|
||||||
|
(2.4 GB) or a 9B model at `q4_k_m` (5.4 GB).
|
||||||
|
|
||||||
|
Start the engine on its own port. The model name given to `--alias` is the name
|
||||||
|
clients will use:
|
||||||
|
|
||||||
|
```
|
||||||
|
llama serve \
|
||||||
|
-m /path/to/Llama-3.2-1B-Instruct-BF16.gguf \
|
||||||
|
--host 0.0.0.0 --port 8082 -ngl 99 --alias llama
|
||||||
|
```
|
||||||
|
|
||||||
|
`-ngl 99` offloads every layer to the GPU. Confirm it is up and knows the model:
|
||||||
|
|
||||||
|
```
|
||||||
|
curl -s localhost:8082/v1/models
|
||||||
|
```
|
||||||
|
|
||||||
|
Then point the router at it. The format is `id=URL=model`:
|
||||||
|
|
||||||
|
```
|
||||||
|
./bin/llmgw server --endpoint e1=http://127.0.0.1:8082=llama
|
||||||
|
```
|
||||||
|
|
||||||
|
## 4. Ask it something
|
||||||
|
|
||||||
|
Streaming:
|
||||||
|
|
||||||
|
```
|
||||||
|
curl -N localhost:8080/v1/chat/completions \
|
||||||
|
-H 'content-type: application/json' \
|
||||||
|
-H 'x-session-id: demo' \
|
||||||
|
-d '{"model":"llama","messages":[{"role":"user","content":"hello"}],"stream":true}'
|
||||||
|
```
|
||||||
|
|
||||||
|
One reply at once:
|
||||||
|
|
||||||
|
```
|
||||||
|
curl -s localhost:8080/v1/chat/completions \
|
||||||
|
-H 'content-type: application/json' \
|
||||||
|
-d '{"model":"llama","messages":[{"role":"user","content":"hello"}],"max_tokens":100}'
|
||||||
|
```
|
||||||
|
|
||||||
|
Also useful: `curl -s localhost:8080/v1/models` and
|
||||||
|
`curl -s localhost:8080/health`. `llmgw ctl` does not work in this mode: it
|
||||||
|
talks to the control plane's admin API, which router-only mode does not have,
|
||||||
|
so `ctl status` answers `404`.
|
||||||
|
|
||||||
|
On the machine this was written on, a 1B bf16 model on an RTX 4070 laptop GPU
|
||||||
|
answered at roughly 85 tokens per second.
|
||||||
|
|
||||||
|
## Several replicas of one model
|
||||||
|
|
||||||
|
Give the router more than one `--endpoint` with the same model name and it
|
||||||
|
spreads requests across them. `x-session-id` keeps a conversation on the same
|
||||||
|
replica so its cache is reused:
|
||||||
|
|
||||||
|
```
|
||||||
|
./bin/llmgw server \
|
||||||
|
--endpoint e1=http://127.0.0.1:8082=llama \
|
||||||
|
--endpoint e2=http://127.0.0.1:8083=llama
|
||||||
|
```
|
||||||
|
|
||||||
|
## Stopping
|
||||||
|
|
||||||
|
```
|
||||||
|
pkill -f "llmgw server"
|
||||||
|
pkill -f "llmgw sim-engine"
|
||||||
|
```
|
||||||
|
|
||||||
|
Stop the engine the way you started it (Ctrl-C in its terminal).
|
||||||
|
|
||||||
|
## What goes wrong
|
||||||
|
|
||||||
|
**`listening on :8080: bind: address already in use`.** An earlier router is
|
||||||
|
still running. `ss -ltnp | grep 8080` names the process; stop it first. This is
|
||||||
|
the usual result of moving from step 2 to step 3 without stopping the fake
|
||||||
|
engine's router.
|
||||||
|
|
||||||
|
**`/v1/models` on the engine returns an empty list.** The engine was started
|
||||||
|
without `-m`, so it is a router with nothing to load. Restart it with a model.
|
||||||
|
|
||||||
|
**`error: nothing to serve`.** `llmgw server` with no `--endpoint` and no model
|
||||||
|
carrying `profiles` has nothing to do. In router-only mode, always pass at least
|
||||||
|
one `--endpoint`.
|
||||||
|
|
||||||
|
**Requests fail with a model-not-found error.** The name in the request must
|
||||||
|
match the name after the last `=` in `--endpoint`, and it must be a name the
|
||||||
|
engine actually serves (its `--alias`).
|
||||||
|
|
||||||
|
**Answers are poor.** A 1B model is a plumbing test, not a chat assistant. Swap
|
||||||
|
in a larger `.gguf`; nothing else changes.
|
||||||
|
|
||||||
|
## 5. Let llmgw run the engine (control plane and node agent)
|
||||||
|
|
||||||
|
Steps 2 and 3 start the engine by hand. Here a **node agent** starts and
|
||||||
|
supervises llama.cpp for you, and a control plane decides where the model runs.
|
||||||
|
Still one machine, still no `sudo`. Stop everything from steps 2 and 3 first:
|
||||||
|
the GPU has to be free and port 8080 unused.
|
||||||
|
|
||||||
|
```
|
||||||
|
curl :8080 -> llmgw server (control plane + router) <- llmgw agent (this node)
|
||||||
|
|
|
||||||
|
starts llama-server :20000
|
||||||
|
```
|
||||||
|
|
||||||
|
The agent dials the control plane over mutual TLS, so this needs a small private
|
||||||
|
certificate authority. Everything goes in a scratch directory, `devcluster/`,
|
||||||
|
which is in `.gitignore`.
|
||||||
|
|
||||||
|
### Certificates
|
||||||
|
|
||||||
|
```
|
||||||
|
mkdir -p devcluster/pki
|
||||||
|
./bin/llmgw ctl cert init-ca --dir devcluster/pki
|
||||||
|
./bin/llmgw ctl cert issue-server llmgw-ctl --host 127.0.0.1 --host localhost \
|
||||||
|
--dir devcluster/pki --out devcluster
|
||||||
|
./bin/llmgw ctl cert issue-client llmgw-node01 --dir devcluster/pki --out devcluster
|
||||||
|
```
|
||||||
|
|
||||||
|
The name given to `issue-client` becomes the certificate's common name and must
|
||||||
|
equal `node_id` in the agent config. The control plane refuses an agent whose
|
||||||
|
claimed id differs from its certificate.
|
||||||
|
|
||||||
|
### The engine wrapper
|
||||||
|
|
||||||
|
The agent starts the engine as `<bin> --model ... --port ...`. A `llama` binary
|
||||||
|
that wants `llama serve ...` refuses every flag, so put a wrapper in front of it:
|
||||||
|
|
||||||
|
```
|
||||||
|
cat > devcluster/llama-server <<'EOF'
|
||||||
|
#!/bin/sh
|
||||||
|
exec /path/to/llama serve "$@"
|
||||||
|
EOF
|
||||||
|
chmod +x devcluster/llama-server
|
||||||
|
|
||||||
|
./bin/llmgw ctl engine-check llamacpp --bin devcluster/llama-server
|
||||||
|
```
|
||||||
|
|
||||||
|
`engine-check` should end with `every flag llmgw builds is accepted`. If your
|
||||||
|
build already ships a `llama-server` binary, point `bin` at that and skip the
|
||||||
|
wrapper.
|
||||||
|
|
||||||
|
### Control plane config: `devcluster/server.yaml`
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
listen: "127.0.0.1:8080"
|
||||||
|
admin_listen: "127.0.0.1:9090"
|
||||||
|
agent_listen: "127.0.0.1:7443"
|
||||||
|
|
||||||
|
agent_tls:
|
||||||
|
ca: devcluster/pki/ca.pem
|
||||||
|
cert: devcluster/server.pem
|
||||||
|
key: devcluster/server-key.pem
|
||||||
|
|
||||||
|
models:
|
||||||
|
- name: llama
|
||||||
|
source: /path/to/Llama-3.2-1B-Instruct-BF16.gguf
|
||||||
|
routing_profile: chat
|
||||||
|
context_length: 8192
|
||||||
|
scaling: { min: 1, max: 1 }
|
||||||
|
profiles:
|
||||||
|
- name: gpu
|
||||||
|
engine: llamacpp
|
||||||
|
gpu_class: 4070-8g
|
||||||
|
weight: 1
|
||||||
|
priority: 100
|
||||||
|
startup_timeout: 5m
|
||||||
|
```
|
||||||
|
|
||||||
|
Three things in there are easy to get wrong:
|
||||||
|
|
||||||
|
- **The model must have at least one profile.** That is what switches the control
|
||||||
|
plane on. With `models: []` the server starts as a plain router and exits with
|
||||||
|
`nothing to serve`.
|
||||||
|
- **`gpu_class` decides CPU or GPU.** Without it the replica is placed in the CPU
|
||||||
|
pool and runs on the CPU. Get the class name from `./bin/llmgw agent inventory`
|
||||||
|
(the `CLASS` column).
|
||||||
|
- **There is no `auth:` block.** Leave it out for a local trial; with auth on,
|
||||||
|
requests need an API key.
|
||||||
|
|
||||||
|
### Agent config: `devcluster/agent.yaml`
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
node_id: llmgw-node01
|
||||||
|
server: "https://127.0.0.1:7443"
|
||||||
|
mgmt_addr: "127.0.0.1"
|
||||||
|
|
||||||
|
tls:
|
||||||
|
ca: devcluster/pki/ca.pem
|
||||||
|
cert: devcluster/llmgw-node01.pem
|
||||||
|
key: devcluster/llmgw-node01-key.pem
|
||||||
|
|
||||||
|
engines:
|
||||||
|
llamacpp:
|
||||||
|
bin: /absolute/path/to/devcluster/llama-server
|
||||||
|
|
||||||
|
supervisor:
|
||||||
|
work_dir: devcluster/agent-work
|
||||||
|
```
|
||||||
|
|
||||||
|
### Run it
|
||||||
|
|
||||||
|
Two terminals:
|
||||||
|
|
||||||
|
```
|
||||||
|
./bin/llmgw server --config devcluster/server.yaml
|
||||||
|
./bin/llmgw agent --config devcluster/agent.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
The model is not placed at once. The control plane waits about 35 seconds for the
|
||||||
|
fleet to settle after it starts, then places the replica, and the agent then
|
||||||
|
loads the weights. Watch it with:
|
||||||
|
|
||||||
|
```
|
||||||
|
./bin/llmgw ctl nodes
|
||||||
|
./bin/llmgw ctl models
|
||||||
|
./bin/llmgw ctl replicas
|
||||||
|
./bin/llmgw ctl explain-placement llama
|
||||||
|
```
|
||||||
|
|
||||||
|
`STATUS ready` in `ctl models` means it is serving. `nvidia-smi` should now show
|
||||||
|
the model in GPU memory. The same curl from step 4 works against port 8080.
|
||||||
|
|
||||||
|
`ctl` works here, unlike in router-only mode, because there is a control plane
|
||||||
|
for it to talk to.
|
||||||
|
|
||||||
|
Stop it with Ctrl-C in each terminal. The agent leaves the engine running when
|
||||||
|
the agent itself stops, so if you restart the agent and the GPU stays full,
|
||||||
|
`pgrep -af "llama serve"` will show the engine still up.
|
||||||
|
|
||||||
|
## 6. Memory: conversations and notes
|
||||||
|
|
||||||
|
Off unless you ask. llmgw otherwise stores nothing anyone typed. On a single
|
||||||
|
machine with no API keys, one block in the server config turns on two things:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
threads:
|
||||||
|
store: devcluster/data/threads.db
|
||||||
|
retain_hours: 24
|
||||||
|
```
|
||||||
|
|
||||||
|
Restart the server after adding it. `retain_hours` is required: everything kept
|
||||||
|
expires that long after it was last written.
|
||||||
|
|
||||||
|
Everything kept this way sits in one shared pool with no owner, and anyone who
|
||||||
|
can reach the port and knows an id can read it. That is fine on one machine and
|
||||||
|
wrong on a shared network. With auth on, threads and notes are per project
|
||||||
|
instead and the block is ignored; see [install.md](install.md).
|
||||||
|
|
||||||
|
### Conversations (threads)
|
||||||
|
|
||||||
|
Send the same `x-llmgw-thread` id and only the new turn. llmgw puts the stored
|
||||||
|
history in front of it:
|
||||||
|
|
||||||
|
```
|
||||||
|
curl -s localhost:8080/v1/chat/completions \
|
||||||
|
-H 'content-type: application/json' -H 'x-llmgw-thread: demo' \
|
||||||
|
-d '{"model":"llama","messages":[{"role":"user","content":"My name is Deniz."}]}'
|
||||||
|
|
||||||
|
curl -s localhost:8080/v1/chat/completions \
|
||||||
|
-H 'content-type: application/json' -H 'x-llmgw-thread: demo' \
|
||||||
|
-d '{"model":"llama","messages":[{"role":"user","content":"What is my name?"}]}'
|
||||||
|
|
||||||
|
curl -X DELETE localhost:8080/v1/threads/demo # forget it
|
||||||
|
```
|
||||||
|
|
||||||
|
Without the block, a request carrying `x-llmgw-thread` is refused with
|
||||||
|
`this gateway keeps no conversations` rather than answered without its history.
|
||||||
|
|
||||||
|
### Notes an agent keeps for itself
|
||||||
|
|
||||||
|
Notes outlive any one conversation. Each is a key, some text and optional tags,
|
||||||
|
kept per agent; name the agent with `x-llmgw-agent` (default `default`), so two
|
||||||
|
agents using the same key do not read each other's notes.
|
||||||
|
|
||||||
|
```
|
||||||
|
curl -X PUT localhost:8080/v1/memory/favorite-color \
|
||||||
|
-H 'content-type: application/json' \
|
||||||
|
-d '{"text":"The user likes dark green.","tags":["preference"]}'
|
||||||
|
|
||||||
|
curl 'localhost:8080/v1/memory?q=green' # search by words
|
||||||
|
curl 'localhost:8080/v1/memory?tag=preference' # or by tag
|
||||||
|
curl localhost:8080/v1/memory/favorite-color
|
||||||
|
curl -X DELETE localhost:8080/v1/memory/favorite-color
|
||||||
|
```
|
||||||
|
|
||||||
|
The same notes are offered to an agent as tools over MCP at
|
||||||
|
`http://localhost:8080/mcp`: `remember`, `recall` and `forget`. Add that URL to
|
||||||
|
your agent's MCP server list (HTTP transport; how you set request headers such
|
||||||
|
as `x-llmgw-agent` depends on the client). llmgw does not put these tools into
|
||||||
|
chat requests on its own, so an agent that is not given the server never sees
|
||||||
|
memory.
|
||||||
|
|
||||||
|
Limits: a key is up to 200 bytes, a note's text up to 16 KiB, at most 16 tags,
|
||||||
|
1000 notes per agent. Tags are lowercase letters, digits, `-` and `_`.
|
||||||
|
|
||||||
|
## Where this stops
|
||||||
|
|
||||||
|
Router-only mode (steps 2 to 4) has no authentication, no placement and no model
|
||||||
|
management: you start and restart engines yourself. Step 5 adds placement, model
|
||||||
|
management and a node agent, and can add authentication as `install.md`
|
||||||
|
describes. For a real cluster, with several nodes, systemd and a persistent
|
||||||
|
certificate authority, follow [install.md](install.md).
|
||||||
|
|
||||||
|
One caveat found while writing this page: `install.md` recommends starting with
|
||||||
|
an empty `models: []` and adding models later with `ctl model add`. The control
|
||||||
|
plane only turns on when the config declares a model with `profiles`, so that
|
||||||
|
first start does not produce a control plane.
|
||||||
@@ -3,6 +3,8 @@ module gitea.mynodes.xyz/ilker/llmgw
|
|||||||
go 1.26.3
|
go 1.26.3
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
connectrpc.com/connect v1.21.0
|
||||||
|
github.com/NVIDIA/go-nvml v0.13.4-0
|
||||||
github.com/cespare/xxhash/v2 v2.3.0
|
github.com/cespare/xxhash/v2 v2.3.0
|
||||||
github.com/knadh/koanf/parsers/yaml v1.1.1
|
github.com/knadh/koanf/parsers/yaml v1.1.1
|
||||||
github.com/knadh/koanf/providers/env/v2 v2.0.1
|
github.com/knadh/koanf/providers/env/v2 v2.0.1
|
||||||
@@ -12,21 +14,33 @@ require (
|
|||||||
github.com/prometheus/client_model v0.6.3
|
github.com/prometheus/client_model v0.6.3
|
||||||
github.com/prometheus/common v0.71.0
|
github.com/prometheus/common v0.71.0
|
||||||
github.com/spf13/cobra v1.10.2
|
github.com/spf13/cobra v1.10.2
|
||||||
|
golang.org/x/net v0.59.0
|
||||||
|
golang.org/x/sync v0.23.0
|
||||||
|
google.golang.org/protobuf v1.36.12
|
||||||
|
modernc.org/sqlite v1.59.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/beorn7/perks v1.0.1 // indirect
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||||
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
github.com/knadh/koanf/maps v0.1.2 // indirect
|
github.com/knadh/koanf/maps v0.1.2 // indirect
|
||||||
|
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||||
github.com/mitchellh/copystructure v1.2.0 // indirect
|
github.com/mitchellh/copystructure v1.2.0 // indirect
|
||||||
github.com/mitchellh/reflectwalk v1.0.2 // indirect
|
github.com/mitchellh/reflectwalk v1.0.2 // indirect
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||||
|
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||||
github.com/prometheus/procfs v0.21.1 // indirect
|
github.com/prometheus/procfs v0.21.1 // indirect
|
||||||
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||||
github.com/spf13/pflag v1.0.9 // indirect
|
github.com/spf13/pflag v1.0.9 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||||
golang.org/x/sync v0.23.0 // indirect
|
golang.org/x/crypto v0.57.0 // indirect
|
||||||
golang.org/x/sys v0.47.0 // indirect
|
golang.org/x/sys v0.48.0 // indirect
|
||||||
google.golang.org/protobuf v1.36.12 // indirect
|
golang.org/x/text v0.42.0 // indirect
|
||||||
|
modernc.org/libc v1.75.7 // indirect
|
||||||
|
modernc.org/mathutil v1.7.1 // indirect
|
||||||
|
modernc.org/memory v1.12.1 // indirect
|
||||||
)
|
)
|
||||||
@@ -1,14 +1,26 @@
|
|||||||
|
connectrpc.com/connect v1.21.0 h1:LhqSJt7jHf5NJBo9Jq/t/9FjcYAideif0mg+qe2jCUs=
|
||||||
|
connectrpc.com/connect v1.21.0/go.mod h1:A2ygJrukXwWy32vkCAAHNVguZrqZ+jeZ9rGRnGR4dN4=
|
||||||
|
github.com/NVIDIA/go-nvml v0.13.4-0 h1:o3jp9u2x1R9ShFE3v+Aesp55XOSIQFMJz/VGNUcJaNE=
|
||||||
|
github.com/NVIDIA/go-nvml v0.13.4-0/go.mod h1:id63qwpoDWpFXwnwM6psDCSqW4BmNu6mWpr4YeQtPGo=
|
||||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||||
|
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||||
|
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||||
|
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
|
||||||
|
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
|
||||||
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
|
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||||
|
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||||
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
|
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
|
||||||
@@ -25,12 +37,16 @@ github.com/knadh/koanf/v2 v2.3.6 h1:JoQPSJmvS4aP0xNc8xMDr5tcrkSEInL23/Il7pITAKo=
|
|||||||
github.com/knadh/koanf/v2 v2.3.6/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28=
|
github.com/knadh/koanf/v2 v2.3.6/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28=
|
||||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||||
|
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||||
|
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||||
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
|
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
|
||||||
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
|
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
|
||||||
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
|
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
|
||||||
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
|
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||||
|
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||||
|
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||||
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
|
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
|
||||||
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
|
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
|
||||||
github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
|
github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
|
||||||
@@ -39,6 +55,8 @@ github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3
|
|||||||
github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=
|
github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=
|
||||||
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
|
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
|
||||||
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
|
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
|
||||||
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||||
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||||
@@ -53,10 +71,48 @@ go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
|||||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||||
|
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||||
|
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||||
|
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
|
||||||
|
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
|
||||||
|
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
|
||||||
|
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||||
|
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||||
|
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||||
|
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
||||||
|
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
||||||
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||||
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8=
|
||||||
|
modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||||
|
modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w=
|
||||||
|
modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I=
|
||||||
|
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||||
|
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||||
|
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||||
|
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||||
|
modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
|
||||||
|
modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||||
|
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||||
|
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||||
|
modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew=
|
||||||
|
modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ=
|
||||||
|
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||||
|
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||||
|
modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
|
||||||
|
modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||||
|
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
|
||||||
|
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||||
|
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||||
|
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||||
|
modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A=
|
||||||
|
modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k=
|
||||||
|
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||||
|
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||||
|
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||||
|
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||||
@@ -0,0 +1,396 @@
|
|||||||
|
package account
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newStore(t *testing.T) *store.SQLite {
|
||||||
|
t.Helper()
|
||||||
|
s, err := store.OpenSQLite(context.Background(), filepath.Join(t.TempDir(), "a.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = s.Close() })
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
for _, err := range []error{
|
||||||
|
s.PutOrg(ctx, store.Org{ID: "org", Name: "Org"}),
|
||||||
|
s.PutTeam(ctx, store.Team{ID: "team", OrgID: "org", Name: "Team"}),
|
||||||
|
s.PutProject(ctx, store.Project{ID: "proj", TeamID: "team", Name: "Proj"}),
|
||||||
|
} {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func newService(t *testing.T) (*Service, *store.SQLite) {
|
||||||
|
t.Helper()
|
||||||
|
s := newStore(t)
|
||||||
|
return New(s, DefaultConfig(), slog.New(slog.DiscardHandler)), s
|
||||||
|
}
|
||||||
|
|
||||||
|
const goodPassword = "correct horse battery staple"
|
||||||
|
|
||||||
|
func TestSignupLeavesAnAccountWaiting(t *testing.T) {
|
||||||
|
// A public signup form that produced a working account would be a way in.
|
||||||
|
svc, st := newService(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
if err := svc.SignUp(ctx, "Ayse@Example.COM", "Ayşe", goodPassword); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
u, err := st.UserByEmail(ctx, "ayse@example.com")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if u.State != store.UserPending {
|
||||||
|
t.Errorf("a new account is %q", u.State)
|
||||||
|
}
|
||||||
|
if u.Email != "ayse@example.com" {
|
||||||
|
t.Errorf("the address was not normalized: %q", u.Email)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, _, err := svc.SignIn(ctx, "ayse@example.com", goodPassword); !errors.Is(err, ErrNotApproved) {
|
||||||
|
t.Errorf("a pending account signed in: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSignupDoesNotRevealExistingAccounts(t *testing.T) {
|
||||||
|
// A form that says "this address already has an account" is an account
|
||||||
|
// enumerator anyone can run, and the person signing up learns nothing they
|
||||||
|
// need: either way they wait for approval.
|
||||||
|
svc, _ := newService(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
if err := svc.SignUp(ctx, "ayse@example.com", "A", goodPassword); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := svc.SignUp(ctx, "ayse@example.com", "Someone Else", goodPassword); err != nil {
|
||||||
|
t.Errorf("the second signup reported %v, which confirms the address is taken", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApprovalIsWhatMakesAnAccountWork(t *testing.T) {
|
||||||
|
svc, st := newService(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
must(t, svc.SignUp(ctx, "ayse@example.com", "A", goodPassword))
|
||||||
|
|
||||||
|
u, _ := st.UserByEmail(ctx, "ayse@example.com")
|
||||||
|
must(t, st.ApproveUser(ctx, u.ID, "proj", "ilker", time.Now()))
|
||||||
|
|
||||||
|
token, csrf, err := svc.SignIn(ctx, "ayse@example.com", goodPassword)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if token == "" || csrf == "" {
|
||||||
|
t.Fatal("sign-in produced no session")
|
||||||
|
}
|
||||||
|
_, got, err := svc.Resolve(ctx, token)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.ProjectID != "proj" {
|
||||||
|
t.Errorf("the approved account landed in project %q", got.ProjectID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApprovalNeedsAProject(t *testing.T) {
|
||||||
|
// An approved account with nowhere to put its keys passes every check and
|
||||||
|
// then fails at the first key it tries to make.
|
||||||
|
_, st := newStoreAndUser(t)
|
||||||
|
if err := st.ApproveUser(context.Background(), "usr-1", "", "ilker", time.Now()); err == nil {
|
||||||
|
t.Error("an account was approved into no project")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newStoreAndUser(t *testing.T) (*Service, *store.SQLite) {
|
||||||
|
t.Helper()
|
||||||
|
svc, st := newService(t)
|
||||||
|
hash, _ := HashPassword(goodPassword)
|
||||||
|
must(t, st.CreateUser(context.Background(), store.User{
|
||||||
|
ID: "usr-1", Email: "ayse@example.com", Name: "A",
|
||||||
|
PasswordHash: hash, State: store.UserPending, CreatedAt: time.Now(),
|
||||||
|
}))
|
||||||
|
return svc, st
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSuspensionTakesEffectOnOpenSessions(t *testing.T) {
|
||||||
|
// Leaving existing sessions open would make suspension advisory.
|
||||||
|
svc, st := newStoreAndUser(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
|
||||||
|
|
||||||
|
token, _, err := svc.SignIn(ctx, "ayse@example.com", goodPassword)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, _, err := svc.Resolve(ctx, token); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
must(t, st.SetUserState(ctx, "usr-1", store.UserSuspended))
|
||||||
|
if _, _, err := svc.Resolve(ctx, token); !errors.Is(err, store.ErrNoSession) {
|
||||||
|
t.Errorf("a suspended account kept its session: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSignOutEndsTheSession(t *testing.T) {
|
||||||
|
svc, st := newStoreAndUser(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
|
||||||
|
|
||||||
|
token, _, _ := svc.SignIn(ctx, "ayse@example.com", goodPassword)
|
||||||
|
must(t, svc.SignOut(ctx, token))
|
||||||
|
if _, _, err := svc.Resolve(ctx, token); !errors.Is(err, store.ErrNoSession) {
|
||||||
|
t.Errorf("the session survived sign-out: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAWrongPasswordAndAMissingAccountAreIndistinguishable(t *testing.T) {
|
||||||
|
svc, st := newStoreAndUser(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
|
||||||
|
|
||||||
|
_, _, wrong := svc.SignIn(ctx, "ayse@example.com", "wrong password here")
|
||||||
|
_, _, missing := svc.SignIn(ctx, "nobody@example.com", "wrong password here")
|
||||||
|
if !errors.Is(wrong, ErrBadCredentials) || !errors.Is(missing, ErrBadCredentials) {
|
||||||
|
t.Fatalf("wrong=%v missing=%v", wrong, missing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGuessingIsBounded(t *testing.T) {
|
||||||
|
// Without this a public login form is an open brute-force target.
|
||||||
|
svc, st := newStoreAndUser(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
|
||||||
|
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.MaxAttempts = 3
|
||||||
|
svc = New(st, cfg, slog.New(slog.DiscardHandler))
|
||||||
|
|
||||||
|
for i := range 3 {
|
||||||
|
if _, _, err := svc.SignIn(ctx, "ayse@example.com", "nope"); !errors.Is(err, ErrBadCredentials) {
|
||||||
|
t.Fatalf("attempt %d: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, _, err := svc.SignIn(ctx, "ayse@example.com", "nope"); !errors.Is(err, ErrTooManyTries) {
|
||||||
|
t.Errorf("guessing was not bounded: %v", err)
|
||||||
|
}
|
||||||
|
// And a correct password is refused too while the window is open, or the
|
||||||
|
// limit would be trivially bypassed by interleaving.
|
||||||
|
if _, _, err := svc.SignIn(ctx, "ayse@example.com", goodPassword); !errors.Is(err, ErrTooManyTries) {
|
||||||
|
t.Errorf("the window did not apply to a correct password: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestASuccessfulSignInClearsTheWindow(t *testing.T) {
|
||||||
|
// Otherwise a person who mistypes their password a few times is locked out
|
||||||
|
// for the rest of the window even after getting it right.
|
||||||
|
st := newStore(t)
|
||||||
|
hash, _ := HashPassword(goodPassword)
|
||||||
|
ctx := context.Background()
|
||||||
|
must(t, st.CreateUser(ctx, store.User{
|
||||||
|
ID: "usr-1", Email: "a@example.com", Name: "A",
|
||||||
|
PasswordHash: hash, State: store.UserPending, CreatedAt: time.Now(),
|
||||||
|
}))
|
||||||
|
must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
|
||||||
|
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.MaxAttempts = 3
|
||||||
|
svc := New(st, cfg, slog.New(slog.DiscardHandler))
|
||||||
|
|
||||||
|
for range 2 {
|
||||||
|
_, _, _ = svc.SignIn(ctx, "a@example.com", "nope")
|
||||||
|
}
|
||||||
|
if _, _, err := svc.SignIn(ctx, "a@example.com", goodPassword); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for range 3 {
|
||||||
|
if _, _, err := svc.SignIn(ctx, "a@example.com", "nope"); errors.Is(err, ErrTooManyTries) {
|
||||||
|
t.Fatal("the window was not cleared by a successful sign-in")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCSRFTokensAreCheckedAgainstTheSession(t *testing.T) {
|
||||||
|
svc, st := newStoreAndUser(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
|
||||||
|
|
||||||
|
token, csrf, _ := svc.SignIn(ctx, "ayse@example.com", goodPassword)
|
||||||
|
sess, _, _ := svc.Resolve(ctx, token)
|
||||||
|
|
||||||
|
if !svc.CheckCSRF(sess, csrf) {
|
||||||
|
t.Error("the session's own token was rejected")
|
||||||
|
}
|
||||||
|
for _, bad := range []string{"", "wrong", csrf + "x", csrf[:len(csrf)-1]} {
|
||||||
|
if svc.CheckCSRF(sess, bad) {
|
||||||
|
t.Errorf("%q was accepted as a csrf token", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A token from another session must not work in this one.
|
||||||
|
other, otherCSRF, _ := svc.SignIn(ctx, "ayse@example.com", goodPassword)
|
||||||
|
_ = other
|
||||||
|
if svc.CheckCSRF(sess, otherCSRF) {
|
||||||
|
t.Error("another session's token was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPasswordsAreHashedNotStored(t *testing.T) {
|
||||||
|
svc, st := newService(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
must(t, svc.SignUp(ctx, "ayse@example.com", "A", goodPassword))
|
||||||
|
|
||||||
|
u, _ := st.UserByEmail(ctx, "ayse@example.com")
|
||||||
|
if strings.Contains(u.PasswordHash, goodPassword) {
|
||||||
|
t.Fatal("the password is in the database")
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(u.PasswordHash, "$argon2id$") {
|
||||||
|
t.Errorf("hash format: %q", u.PasswordHash)
|
||||||
|
}
|
||||||
|
ok, err := VerifyPassword(goodPassword, u.PasswordHash)
|
||||||
|
if err != nil || !ok {
|
||||||
|
t.Errorf("the hash does not verify: %v %v", ok, err)
|
||||||
|
}
|
||||||
|
if ok, _ := VerifyPassword("something else entirely", u.PasswordHash); ok {
|
||||||
|
t.Error("a wrong password verified")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTwoHashesOfOnePasswordDiffer(t *testing.T) {
|
||||||
|
// A salt that did not vary would make the table a rainbow-table lookup.
|
||||||
|
a, _ := HashPassword(goodPassword)
|
||||||
|
b, _ := HashPassword(goodPassword)
|
||||||
|
if a == b {
|
||||||
|
t.Fatal("hashing is not salted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestShortPasswordsAreRefused(t *testing.T) {
|
||||||
|
if _, err := HashPassword("short"); !errors.Is(err, ErrWeakPassword) {
|
||||||
|
t.Errorf("a five-character password was accepted: %v", err)
|
||||||
|
}
|
||||||
|
svc, _ := newService(t)
|
||||||
|
if err := svc.SignUp(context.Background(), "a@example.com", "A", "short"); !errors.Is(err, ErrWeakPassword) {
|
||||||
|
t.Errorf("signup accepted it: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMalformedHashesAreRejectedRatherThanMatched(t *testing.T) {
|
||||||
|
for _, bad := range []string{"", "plaintext", "$argon2id$", "$bcrypt$v=19$m=1,t=1,p=1$aa$bb"} {
|
||||||
|
if ok, err := VerifyPassword(goodPassword, bad); ok || err == nil {
|
||||||
|
t.Errorf("VerifyPassword(%q) = %v, %v", bad, ok, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestObviousNonAddressesAreRefused(t *testing.T) {
|
||||||
|
svc, _ := newService(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
for _, bad := range []string{"", "ayse", "ayse@", "@example.com", "a b@example.com", "a@b", "a@@b.com"} {
|
||||||
|
if err := svc.SignUp(ctx, bad, "A", goodPassword); !errors.Is(err, ErrInvalidEmail) {
|
||||||
|
t.Errorf("SignUp(%q) = %v", bad, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func must(t *testing.T, err error) {
|
||||||
|
t.Helper()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChangingAPasswordNeedsTheOldOne(t *testing.T) {
|
||||||
|
// A session is what an attacker gets from a borrowed laptop or a stolen
|
||||||
|
// cookie. The password is what they do not have, and without this check the
|
||||||
|
// first thing they would do is take the account.
|
||||||
|
svc, st := newStoreAndUser(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
|
||||||
|
u, _ := st.UserByID(ctx, "usr-1")
|
||||||
|
|
||||||
|
if err := svc.ChangePassword(ctx, st, u, "not the password", "a brand new passphrase"); !errors.Is(err, ErrBadCredentials) {
|
||||||
|
t.Errorf("a wrong current password was accepted: %v", err)
|
||||||
|
}
|
||||||
|
// And the old one still works.
|
||||||
|
if _, _, err := svc.SignIn(ctx, "ayse@example.com", goodPassword); err != nil {
|
||||||
|
t.Errorf("the password changed anyway: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChangingAPasswordSignsOutEverywhere(t *testing.T) {
|
||||||
|
// Leaving other sessions open would mean a password change does nothing
|
||||||
|
// about the reason someone is changing it.
|
||||||
|
svc, st := newStoreAndUser(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
|
||||||
|
|
||||||
|
laptop, _, _ := svc.SignIn(ctx, "ayse@example.com", goodPassword)
|
||||||
|
phone, _, _ := svc.SignIn(ctx, "ayse@example.com", goodPassword)
|
||||||
|
|
||||||
|
u, _ := st.UserByID(ctx, "usr-1")
|
||||||
|
must(t, svc.ChangePassword(ctx, st, u, goodPassword, "a brand new passphrase"))
|
||||||
|
|
||||||
|
for name, token := range map[string]string{"laptop": laptop, "phone": phone} {
|
||||||
|
if _, _, err := svc.Resolve(ctx, token); !errors.Is(err, store.ErrNoSession) {
|
||||||
|
t.Errorf("the %s session survived the change: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The new password works and the old one does not.
|
||||||
|
if _, _, err := svc.SignIn(ctx, "ayse@example.com", "a brand new passphrase"); err != nil {
|
||||||
|
t.Errorf("the new password does not work: %v", err)
|
||||||
|
}
|
||||||
|
if _, _, err := svc.SignIn(ctx, "ayse@example.com", goodPassword); !errors.Is(err, ErrBadCredentials) {
|
||||||
|
t.Errorf("the old password still works: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestANewPasswordMustActuallyBeNew(t *testing.T) {
|
||||||
|
svc, st := newStoreAndUser(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
|
||||||
|
u, _ := st.UserByID(ctx, "usr-1")
|
||||||
|
|
||||||
|
if err := svc.ChangePassword(ctx, st, u, goodPassword, goodPassword); !errors.Is(err, ErrSamePassword) {
|
||||||
|
t.Errorf("changing a password to itself: %v", err)
|
||||||
|
}
|
||||||
|
if err := svc.ChangePassword(ctx, st, u, goodPassword, "short"); !errors.Is(err, ErrWeakPassword) {
|
||||||
|
t.Errorf("a short new password was accepted: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGuessingTheCurrentPasswordIsBounded(t *testing.T) {
|
||||||
|
// Someone with a stolen session should not get unlimited attempts at the
|
||||||
|
// password that would let them keep it.
|
||||||
|
st := newStore(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
hash, _ := HashPassword(goodPassword)
|
||||||
|
must(t, st.CreateUser(ctx, store.User{
|
||||||
|
ID: "usr-1", Email: "a@example.com", Name: "A",
|
||||||
|
PasswordHash: hash, State: store.UserPending, CreatedAt: time.Now(),
|
||||||
|
}))
|
||||||
|
must(t, st.ApproveUser(ctx, "usr-1", "proj", "ilker", time.Now()))
|
||||||
|
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.MaxAttempts = 3
|
||||||
|
svc := New(st, cfg, slog.New(slog.DiscardHandler))
|
||||||
|
u, _ := st.UserByID(ctx, "usr-1")
|
||||||
|
|
||||||
|
for range 3 {
|
||||||
|
_ = svc.ChangePassword(ctx, st, u, "wrong", "a brand new passphrase")
|
||||||
|
}
|
||||||
|
if err := svc.ChangePassword(ctx, st, u, "wrong", "a brand new passphrase"); !errors.Is(err, ErrTooManyTries) {
|
||||||
|
t.Errorf("guessing was unbounded: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
// Package account is the human half of authentication: people sign up, an
|
||||||
|
// operator approves them, and they issue their own API keys from a profile page
|
||||||
|
// (plan §7.9).
|
||||||
|
//
|
||||||
|
// This is deliberately a different mechanism from the one in internal/auth.
|
||||||
|
// That one authenticates machines holding 256-bit random keys; this one
|
||||||
|
// authenticates people holding passwords they chose. The two have opposite
|
||||||
|
// requirements — a fast hash is correct there and wrong here — and keeping them
|
||||||
|
// apart is what stops one set of assumptions leaking into the other.
|
||||||
|
package account
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/subtle"
|
||||||
|
"encoding/base64"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/argon2"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Argon2 parameters. OWASP's current floor for argon2id, which costs roughly
|
||||||
|
// 50ms per attempt on ordinary hardware — slow enough to make an offline
|
||||||
|
// dictionary attack expensive, fast enough that a login does not feel broken.
|
||||||
|
//
|
||||||
|
// They are stored with each hash, so raising them later leaves existing
|
||||||
|
// passwords working until their owners next sign in.
|
||||||
|
const (
|
||||||
|
argonTime = 3
|
||||||
|
argonMemory = 64 * 1024 // KiB
|
||||||
|
argonThreads = 4
|
||||||
|
argonKeyLen = 32
|
||||||
|
saltLen = 16
|
||||||
|
)
|
||||||
|
|
||||||
|
// MinPasswordLen is the only password rule.
|
||||||
|
//
|
||||||
|
// Length is the one requirement that reliably buys entropy. Composition rules —
|
||||||
|
// a digit, a symbol, a capital — push people towards Password1! and produce
|
||||||
|
// less entropy than they cost, so there are none.
|
||||||
|
const MinPasswordLen = 12
|
||||||
|
|
||||||
|
// ErrWeakPassword says a password is too short to accept.
|
||||||
|
var ErrWeakPassword = fmt.Errorf("a password must be at least %d characters", MinPasswordLen)
|
||||||
|
|
||||||
|
// ErrBadHash says a stored hash could not be parsed.
|
||||||
|
var ErrBadHash = errors.New("unreadable password hash")
|
||||||
|
|
||||||
|
// HashPassword derives a storable hash.
|
||||||
|
func HashPassword(password string) (string, error) {
|
||||||
|
if len([]rune(password)) < MinPasswordLen {
|
||||||
|
return "", ErrWeakPassword
|
||||||
|
}
|
||||||
|
salt := make([]byte, saltLen)
|
||||||
|
if _, err := rand.Read(salt); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
sum := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen)
|
||||||
|
|
||||||
|
b64 := base64.RawStdEncoding.EncodeToString
|
||||||
|
return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s",
|
||||||
|
argon2.Version, argonMemory, argonTime, argonThreads, b64(salt), b64(sum)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// VerifyPassword reports whether a password matches a stored hash.
|
||||||
|
//
|
||||||
|
// The comparison is constant time. The parameters come from the hash rather
|
||||||
|
// than from the constants above, so a hash written under older settings still
|
||||||
|
// verifies.
|
||||||
|
func VerifyPassword(password, encoded string) (bool, error) {
|
||||||
|
parts := strings.Split(encoded, "$")
|
||||||
|
if len(parts) != 6 || parts[1] != "argon2id" {
|
||||||
|
return false, ErrBadHash
|
||||||
|
}
|
||||||
|
|
||||||
|
var version int
|
||||||
|
if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil || version != argon2.Version {
|
||||||
|
return false, ErrBadHash
|
||||||
|
}
|
||||||
|
var memory, time uint32
|
||||||
|
var threads uint8
|
||||||
|
if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &memory, &time, &threads); err != nil {
|
||||||
|
return false, ErrBadHash
|
||||||
|
}
|
||||||
|
|
||||||
|
salt, err := base64.RawStdEncoding.DecodeString(parts[4])
|
||||||
|
if err != nil {
|
||||||
|
return false, ErrBadHash
|
||||||
|
}
|
||||||
|
want, err := base64.RawStdEncoding.DecodeString(parts[5])
|
||||||
|
if err != nil {
|
||||||
|
return false, ErrBadHash
|
||||||
|
}
|
||||||
|
|
||||||
|
got := argon2.IDKey([]byte(password), salt, time, memory, threads, uint32(len(want)))
|
||||||
|
return subtle.ConstantTimeCompare(got, want) == 1, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// dummyHash is verified against when no account matches, so a login attempt for
|
||||||
|
// an address that does not exist costs the same time as one that does.
|
||||||
|
//
|
||||||
|
// Without it, a stopwatch enumerates every account on the system.
|
||||||
|
var dummyHash string
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
h, err := HashPassword(strings.Repeat("x", MinPasswordLen))
|
||||||
|
if err != nil {
|
||||||
|
panic("account: cannot build the dummy hash: " + err.Error())
|
||||||
|
}
|
||||||
|
dummyHash = h
|
||||||
|
}
|
||||||
@@ -0,0 +1,330 @@
|
|||||||
|
package account
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/subtle"
|
||||||
|
"encoding/base32"
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What a sign-in or sign-up can fail with.
|
||||||
|
var (
|
||||||
|
ErrBadCredentials = errors.New("that email and password do not match an account")
|
||||||
|
ErrNotApproved = errors.New("this account is waiting for approval")
|
||||||
|
ErrSuspended = errors.New("this account has been suspended")
|
||||||
|
ErrTooManyTries = errors.New("too many attempts; try again shortly")
|
||||||
|
ErrInvalidEmail = errors.New("that does not look like an email address")
|
||||||
|
)
|
||||||
|
|
||||||
|
// Store is what the service needs from persistence.
|
||||||
|
type Store interface {
|
||||||
|
CreateUser(ctx context.Context, u store.User) error
|
||||||
|
UserByEmail(ctx context.Context, email string) (store.User, error)
|
||||||
|
UserByID(ctx context.Context, id string) (store.User, error)
|
||||||
|
TouchLogin(ctx context.Context, id string, at time.Time) error
|
||||||
|
|
||||||
|
CreateSession(ctx context.Context, userID, token, csrf string, ttl time.Duration) error
|
||||||
|
SessionByToken(ctx context.Context, token string) (store.Session, store.User, error)
|
||||||
|
DeleteSession(ctx context.Context, token string) error
|
||||||
|
TouchSession(ctx context.Context, token string, at time.Time) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config tunes the service.
|
||||||
|
type Config struct {
|
||||||
|
// SignupOpen decides whether anyone may create an account. Off, the form is
|
||||||
|
// not served at all; accounts are made with `llmgw users create`.
|
||||||
|
SignupOpen bool
|
||||||
|
// SessionTTL is how long a sign-in lasts.
|
||||||
|
SessionTTL time.Duration
|
||||||
|
// MaxAttempts and AttemptWindow bound password guessing per address.
|
||||||
|
MaxAttempts int
|
||||||
|
AttemptWindow time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultConfig returns sensible values.
|
||||||
|
func DefaultConfig() Config {
|
||||||
|
return Config{
|
||||||
|
SignupOpen: true,
|
||||||
|
SessionTTL: store.SessionTTL,
|
||||||
|
MaxAttempts: 10,
|
||||||
|
AttemptWindow: 15 * time.Minute,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) normalize() {
|
||||||
|
d := DefaultConfig()
|
||||||
|
if c.SessionTTL <= 0 {
|
||||||
|
c.SessionTTL = d.SessionTTL
|
||||||
|
}
|
||||||
|
if c.MaxAttempts <= 0 {
|
||||||
|
c.MaxAttempts = d.MaxAttempts
|
||||||
|
}
|
||||||
|
if c.AttemptWindow <= 0 {
|
||||||
|
c.AttemptWindow = d.AttemptWindow
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Service is signup, sign-in and sessions.
|
||||||
|
type Service struct {
|
||||||
|
store Store
|
||||||
|
cfg Config
|
||||||
|
log *slog.Logger
|
||||||
|
now func() time.Time
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
attempts map[string]*attemptWindow
|
||||||
|
}
|
||||||
|
|
||||||
|
type attemptWindow struct {
|
||||||
|
count int
|
||||||
|
since time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// New builds the service.
|
||||||
|
func New(s Store, cfg Config, log *slog.Logger) *Service {
|
||||||
|
cfg.normalize()
|
||||||
|
if log == nil {
|
||||||
|
log = slog.Default()
|
||||||
|
}
|
||||||
|
return &Service{store: s, cfg: cfg, log: log, now: time.Now, attempts: map[string]*attemptWindow{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SignupOpen reports whether the form should be served.
|
||||||
|
func (s *Service) SignupOpen() bool { return s.cfg.SignupOpen }
|
||||||
|
|
||||||
|
// SignUp creates a pending account.
|
||||||
|
//
|
||||||
|
// It returns nil whether or not the address was already taken. A signup form
|
||||||
|
// that says "this address already has an account" is an account enumerator that
|
||||||
|
// anyone can run, and the person signing up learns nothing they need: either
|
||||||
|
// way, what happens next is that they wait for approval.
|
||||||
|
func (s *Service) SignUp(ctx context.Context, email, name, password string) error {
|
||||||
|
email = store.NormalizeEmail(email)
|
||||||
|
if !validEmail(email) {
|
||||||
|
return ErrInvalidEmail
|
||||||
|
}
|
||||||
|
hash, err := HashPassword(password)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if name = strings.TrimSpace(name); name == "" {
|
||||||
|
name = email
|
||||||
|
}
|
||||||
|
|
||||||
|
err = s.store.CreateUser(ctx, store.User{
|
||||||
|
ID: randomID("usr"), Email: email, Name: name,
|
||||||
|
PasswordHash: hash, State: store.UserPending, CreatedAt: s.now(),
|
||||||
|
})
|
||||||
|
if errors.Is(err, store.ErrEmailTaken) {
|
||||||
|
// Logged so an operator can see it; not told to the caller.
|
||||||
|
s.log.Info("signup for an address that already has an account")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.log.Info("a new account is waiting for approval", "email", email)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SignIn checks a password and opens a session, returning the session token and
|
||||||
|
// the CSRF token that must accompany the forms in it.
|
||||||
|
func (s *Service) SignIn(ctx context.Context, email, password string) (token, csrf string, err error) {
|
||||||
|
email = store.NormalizeEmail(email)
|
||||||
|
if !s.allowAttempt(email) {
|
||||||
|
return "", "", ErrTooManyTries
|
||||||
|
}
|
||||||
|
|
||||||
|
u, err := s.store.UserByEmail(ctx, email)
|
||||||
|
if errors.Is(err, store.ErrNotFound) {
|
||||||
|
// Verify anyway, against a hash that cannot match. Returning early here
|
||||||
|
// would let a stopwatch enumerate every account on the system.
|
||||||
|
_, _ = VerifyPassword(password, dummyHash)
|
||||||
|
return "", "", ErrBadCredentials
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
ok, err := VerifyPassword(password, u.PasswordHash)
|
||||||
|
if err != nil || !ok {
|
||||||
|
return "", "", ErrBadCredentials
|
||||||
|
}
|
||||||
|
// The password was right, so this was not an attack.
|
||||||
|
s.clearAttempts(email)
|
||||||
|
|
||||||
|
// State is checked after the password, on purpose. Checking first would
|
||||||
|
// tell anyone who guesses an address whether it has a pending account.
|
||||||
|
switch u.State {
|
||||||
|
case store.UserPending:
|
||||||
|
return "", "", ErrNotApproved
|
||||||
|
case store.UserSuspended:
|
||||||
|
return "", "", ErrSuspended
|
||||||
|
}
|
||||||
|
|
||||||
|
if token, err = store.NewSessionToken(); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
if csrf, err = store.NewSessionToken(); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
if err := s.store.CreateSession(ctx, u.ID, token, csrf, s.cfg.SessionTTL); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
if err := s.store.TouchLogin(ctx, u.ID, s.now()); err != nil {
|
||||||
|
s.log.Warn("could not record a sign-in", "err", err)
|
||||||
|
}
|
||||||
|
return token, csrf, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve turns a session cookie into the person holding it.
|
||||||
|
func (s *Service) Resolve(ctx context.Context, token string) (store.Session, store.User, error) {
|
||||||
|
if token == "" {
|
||||||
|
return store.Session{}, store.User{}, store.ErrNoSession
|
||||||
|
}
|
||||||
|
sess, u, err := s.store.SessionByToken(ctx, token)
|
||||||
|
if err != nil {
|
||||||
|
return store.Session{}, store.User{}, err
|
||||||
|
}
|
||||||
|
if err := s.store.TouchSession(ctx, token, s.now()); err != nil {
|
||||||
|
s.log.Debug("could not touch a session", "err", err)
|
||||||
|
}
|
||||||
|
return sess, u, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SignOut closes one session.
|
||||||
|
func (s *Service) SignOut(ctx context.Context, token string) error {
|
||||||
|
if token == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return s.store.DeleteSession(ctx, token)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckCSRF compares a submitted token against the session's.
|
||||||
|
//
|
||||||
|
// A cookie authenticates the browser, not the page that asked. Without this, any
|
||||||
|
// site the person visits can post a form to ours and have the browser attach the
|
||||||
|
// session automatically.
|
||||||
|
func (s *Service) CheckCSRF(sess store.Session, submitted string) bool {
|
||||||
|
if submitted == "" || len(sess.CSRFHash) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return subtle.ConstantTimeCompare(store.HashKey(submitted), sess.CSRFHash) == 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// allowAttempt bounds password guessing per address.
|
||||||
|
//
|
||||||
|
// Per address rather than per IP: an attacker has many addresses to come from
|
||||||
|
// and only one to guess against, and limiting by IP would lock out everyone
|
||||||
|
// behind one office NAT the moment somebody fat-fingers their password.
|
||||||
|
func (s *Service) allowAttempt(email string) bool {
|
||||||
|
now := s.now()
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
w, ok := s.attempts[email]
|
||||||
|
if !ok || now.Sub(w.since) > s.cfg.AttemptWindow {
|
||||||
|
s.attempts[email] = &attemptWindow{count: 1, since: now}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
w.count++
|
||||||
|
return w.count <= s.cfg.MaxAttempts
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) clearAttempts(email string) {
|
||||||
|
s.mu.Lock()
|
||||||
|
delete(s.attempts, email)
|
||||||
|
s.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// validEmail is deliberately crude. A real check is delivery, which this does
|
||||||
|
// not do; this only rejects what is obviously not an address so the database
|
||||||
|
// does not fill with them.
|
||||||
|
func validEmail(s string) bool {
|
||||||
|
at := strings.IndexByte(s, '@')
|
||||||
|
if at <= 0 || at != strings.LastIndexByte(s, '@') || at == len(s)-1 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if strings.ContainsAny(s, " \t\r\n<>\"") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return strings.Contains(s[at+1:], ".")
|
||||||
|
}
|
||||||
|
|
||||||
|
func randomID(prefix string) string {
|
||||||
|
var buf [10]byte
|
||||||
|
_, _ = rand.Read(buf[:])
|
||||||
|
return prefix + "-" + strings.ToLower(
|
||||||
|
base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(buf[:]))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrSamePassword says the new password is the old one.
|
||||||
|
var ErrSamePassword = errors.New("that is the password you already have")
|
||||||
|
|
||||||
|
// PasswordChanger is the extra storage a password change needs.
|
||||||
|
type PasswordChanger interface {
|
||||||
|
SetPassword(ctx context.Context, userID, hash string) error
|
||||||
|
DeleteUserSessions(ctx context.Context, userID string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChangePassword replaces a person's password and signs them out everywhere.
|
||||||
|
//
|
||||||
|
// The current password is required even though the caller already holds a
|
||||||
|
// session. A session is what an attacker gets from a borrowed laptop or a stolen
|
||||||
|
// cookie; the password is what they do not have, and without this check the
|
||||||
|
// first thing they would do is take the account.
|
||||||
|
//
|
||||||
|
// Every session is closed, this browser's included. Leaving the others open
|
||||||
|
// would mean a password change does nothing about the reason someone is
|
||||||
|
// changing it. The caller re-issues a session for the browser in front of them.
|
||||||
|
func (s *Service) ChangePassword(ctx context.Context, pw PasswordChanger,
|
||||||
|
u store.User, current, next string,
|
||||||
|
) error {
|
||||||
|
if !s.allowAttempt("pw:" + u.ID) {
|
||||||
|
return ErrTooManyTries
|
||||||
|
}
|
||||||
|
ok, err := VerifyPassword(current, u.PasswordHash)
|
||||||
|
if err != nil || !ok {
|
||||||
|
return ErrBadCredentials
|
||||||
|
}
|
||||||
|
s.clearAttempts("pw:" + u.ID)
|
||||||
|
|
||||||
|
if same, _ := VerifyPassword(next, u.PasswordHash); same {
|
||||||
|
return ErrSamePassword
|
||||||
|
}
|
||||||
|
hash, err := HashPassword(next)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := pw.SetPassword(ctx, u.ID, hash); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := pw.DeleteUserSessions(ctx, u.ID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.log.Info("a password was changed", "user", u.ID)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenSession starts a session for a person who has just proved who they are by
|
||||||
|
// some other means — immediately after changing their password, which closed
|
||||||
|
// the one they were using.
|
||||||
|
func (s *Service) OpenSession(ctx context.Context, userID string) (token, csrf string, err error) {
|
||||||
|
if token, err = store.NewSessionToken(); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
if csrf, err = store.NewSessionToken(); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
if err := s.store.CreateSession(ctx, userID, token, csrf, s.cfg.SessionTTL); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
return token, csrf, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
package accounting
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Refusals a key can hit. They are distinguished so the client is told which
|
||||||
|
// limit it hit, and so retrying is only suggested where retrying helps.
|
||||||
|
var (
|
||||||
|
ErrRateLimited = errors.New("request rate limit exceeded")
|
||||||
|
ErrTokenLimited = errors.New("token rate limit exceeded")
|
||||||
|
ErrBudgetSpent = errors.New("budget exhausted")
|
||||||
|
)
|
||||||
|
|
||||||
|
// LimitError carries which limit was hit and, where it makes sense, when to
|
||||||
|
// come back.
|
||||||
|
type LimitError struct {
|
||||||
|
Err error
|
||||||
|
// RetryAfter is zero when waiting does not help, which is the case for a
|
||||||
|
// spent budget: it will not refill until the period rolls over, and telling
|
||||||
|
// a client to retry in a second would be a lie.
|
||||||
|
RetryAfter time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *LimitError) Error() string { return e.Err.Error() }
|
||||||
|
func (e *LimitError) Unwrap() error { return e.Err }
|
||||||
|
|
||||||
|
// Accountant enforces limits and records what was used.
|
||||||
|
//
|
||||||
|
// One object because the three halves are not independent: a budget check needs
|
||||||
|
// the price, the price needs the model, and the record closes the loop by
|
||||||
|
// feeding the counters that the next check reads.
|
||||||
|
type Accountant struct {
|
||||||
|
pricing *Pricing
|
||||||
|
budgets *Budgets
|
||||||
|
requests *Limiter
|
||||||
|
tokens *Limiter
|
||||||
|
recorder *Recorder
|
||||||
|
log *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// New builds an accountant.
|
||||||
|
func New(pricing *Pricing, recorder *Recorder, log *slog.Logger) *Accountant {
|
||||||
|
if log == nil {
|
||||||
|
log = slog.Default()
|
||||||
|
}
|
||||||
|
return &Accountant{
|
||||||
|
pricing: pricing,
|
||||||
|
budgets: NewBudgets(),
|
||||||
|
requests: NewLimiter(),
|
||||||
|
tokens: NewLimiter(),
|
||||||
|
recorder: recorder,
|
||||||
|
log: log,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RestoreBudgets seeds the in-memory counters from the store.
|
||||||
|
//
|
||||||
|
// Called at startup. A restart must not reset a budget, which it would if the
|
||||||
|
// counters only ever lived in memory.
|
||||||
|
func (a *Accountant) RestoreBudgets(ctx context.Context, s store.Store, period store.Period) error {
|
||||||
|
start := PeriodStart(period, time.Now())
|
||||||
|
if start.IsZero() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
spend, err := s.SpendSince(ctx, start)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("restoring budgets: %w", err)
|
||||||
|
}
|
||||||
|
a.budgets.Restore(period, spend)
|
||||||
|
a.log.Info("restored budgets from the store", "keys", len(spend), "since", start)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reservation is what Check returns: the estimate it charged, so Record can
|
||||||
|
// settle the difference against the real figure.
|
||||||
|
type Reservation struct {
|
||||||
|
KeyID string
|
||||||
|
EstimatedTokens int64
|
||||||
|
Price Price
|
||||||
|
// Period is the key's budget period, carried from the check so the record
|
||||||
|
// counts against the same window the check read. Deriving it later would
|
||||||
|
// mean guessing, and a monthly budget credited to a daily counter is a
|
||||||
|
// budget that never binds.
|
||||||
|
Period store.Period
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check decides whether a request may proceed.
|
||||||
|
//
|
||||||
|
// It runs before scheduling and touches no database: every counter it reads is
|
||||||
|
// in memory. The token limit is checked against an estimate, because the real
|
||||||
|
// figure only exists once the request has run; Record settles the difference.
|
||||||
|
func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64) (Reservation, error) {
|
||||||
|
price := a.pricing.For(model)
|
||||||
|
res := Reservation{
|
||||||
|
KeyID: p.KeyID, EstimatedTokens: estimatedTokens,
|
||||||
|
Price: price, Period: p.Limits.BudgetPeriod,
|
||||||
|
}
|
||||||
|
if p.KeyID == "" {
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
lim := p.Limits
|
||||||
|
|
||||||
|
// Budget first. It is the limit whose refusal cannot be retried away, so
|
||||||
|
// telling the client about it first is more useful than reporting a rate
|
||||||
|
// limit it would hit again in a second.
|
||||||
|
//
|
||||||
|
// A model that costs nothing is exempt, which is what lets a key with no
|
||||||
|
// budget left still be served a cheap fallback (plan §7.7).
|
||||||
|
if lim.Budget != nil && !price.Zero() &&
|
||||||
|
a.budgets.Exhausted(p.KeyID, lim.BudgetPeriod, *lim.Budget) {
|
||||||
|
return res, &LimitError{Err: ErrBudgetSpent}
|
||||||
|
}
|
||||||
|
|
||||||
|
if lim.RPM != nil && !a.requests.Allow(p.KeyID, *lim.RPM, 1) {
|
||||||
|
return res, &LimitError{Err: ErrRateLimited, RetryAfter: time.Second}
|
||||||
|
}
|
||||||
|
if lim.TPM != nil && !a.tokens.Allow(p.KeyID, *lim.TPM, estimatedTokens) {
|
||||||
|
// The request was not charged against the request-rate limit in vain:
|
||||||
|
// it was, and that is correct — it did arrive.
|
||||||
|
return res, &LimitError{Err: ErrTokenLimited, RetryAfter: time.Second}
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Record settles a finished request: it reconciles the token estimate, adds to
|
||||||
|
// the budget, and queues the usage row.
|
||||||
|
func (a *Accountant) Record(res Reservation, u store.Usage) {
|
||||||
|
if res.KeyID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
actual := u.PromptTokens + u.CompletionTokens
|
||||||
|
|
||||||
|
// The estimate was charged up front; charge the difference now. A caller
|
||||||
|
// whose requests are consistently larger than estimated is still bounded,
|
||||||
|
// one window later than with a perfect estimate.
|
||||||
|
a.tokens.Charge(res.KeyID, actual-res.EstimatedTokens)
|
||||||
|
|
||||||
|
u.CostMicros = res.Price.Cost(u.PromptTokens, u.CompletionTokens)
|
||||||
|
a.budgets.Add(res.KeyID, res.Period, u.CostMicros)
|
||||||
|
|
||||||
|
if a.recorder != nil {
|
||||||
|
a.recorder.Record(u)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status returns what a key has used, for headers and `ctl`.
|
||||||
|
func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) {
|
||||||
|
if p.KeyID == "" {
|
||||||
|
return 0, 0, 0
|
||||||
|
}
|
||||||
|
return a.budgets.Spent(p.KeyID, p.Limits.BudgetPeriod),
|
||||||
|
a.requests.Rate(p.KeyID), a.tokens.Rate(p.KeyID)
|
||||||
|
}
|
||||||
@@ -0,0 +1,290 @@
|
|||||||
|
package accounting
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func intp(v int) *int { return &v }
|
||||||
|
func i64p(v int64) *int64 { return &v }
|
||||||
|
|
||||||
|
func principal(id string, tune func(*store.Effective)) auth.Principal {
|
||||||
|
e := store.Effective{KeyID: id, ProjectID: "proj"}
|
||||||
|
if tune != nil {
|
||||||
|
tune(&e)
|
||||||
|
}
|
||||||
|
return auth.Principal{KeyID: id, ProjectID: "proj", Limits: e}
|
||||||
|
}
|
||||||
|
|
||||||
|
func pricing(t *testing.T) *Pricing {
|
||||||
|
t.Helper()
|
||||||
|
return NewPricing(map[string]Price{
|
||||||
|
"paid": {PromptPer1k: 1000, CompletionPer1k: 3000},
|
||||||
|
"free": {},
|
||||||
|
}, Price{})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCostRoundsUpSoSmallRequestsAreNotFree(t *testing.T) {
|
||||||
|
// Rounding down would make a stream of tiny requests cost nothing, which is
|
||||||
|
// exactly the traffic shape a quota exists to bound.
|
||||||
|
p := Price{PromptPer1k: 1000, CompletionPer1k: 3000}
|
||||||
|
if got := p.Cost(1, 0); got != 1 {
|
||||||
|
t.Errorf("one prompt token cost %d", got)
|
||||||
|
}
|
||||||
|
if got := p.Cost(1000, 1000); got != 4000 {
|
||||||
|
t.Errorf("a thousand of each cost %d, want 4000", got)
|
||||||
|
}
|
||||||
|
if got := p.Cost(0, 0); got != 0 {
|
||||||
|
t.Errorf("an empty request cost %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestZeroPricedModelsStayAvailableWhenTheBudgetIsSpent(t *testing.T) {
|
||||||
|
// This is what makes budget fallback work: a key with nothing left can
|
||||||
|
// still be served something that costs nothing.
|
||||||
|
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||||
|
p := principal("k1", func(e *store.Effective) {
|
||||||
|
e.Budget = i64p(100)
|
||||||
|
e.BudgetPeriod = store.PeriodMonth
|
||||||
|
})
|
||||||
|
|
||||||
|
res, err := a.Check(p, "paid", 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000, CompletionTokens: 0})
|
||||||
|
|
||||||
|
if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) {
|
||||||
|
t.Errorf("a spent budget still served a paid model: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := a.Check(p, "free", 10); err != nil {
|
||||||
|
t.Errorf("a spent budget refused a free model: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBudgetRefusalDoesNotSuggestRetrying(t *testing.T) {
|
||||||
|
// A budget does not refill until the period rolls over, so a Retry-After of
|
||||||
|
// a second would be a lie that turns one client into a retry loop.
|
||||||
|
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||||
|
p := principal("k1", func(e *store.Effective) {
|
||||||
|
e.Budget = i64p(1)
|
||||||
|
e.BudgetPeriod = store.PeriodDay
|
||||||
|
})
|
||||||
|
res, _ := a.Check(p, "paid", 10)
|
||||||
|
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 5000})
|
||||||
|
|
||||||
|
_, err := a.Check(p, "paid", 10)
|
||||||
|
var le *LimitError
|
||||||
|
if !errors.As(err, &le) {
|
||||||
|
t.Fatalf("expected a limit error, got %v", err)
|
||||||
|
}
|
||||||
|
if le.RetryAfter != 0 {
|
||||||
|
t.Errorf("a spent budget suggested retrying in %v", le.RetryAfter)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRateLimitsBind(t *testing.T) {
|
||||||
|
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||||
|
p := principal("k1", func(e *store.Effective) { e.RPM = intp(3) })
|
||||||
|
|
||||||
|
for i := range 3 {
|
||||||
|
if _, err := a.Check(p, "free", 1); err != nil {
|
||||||
|
t.Fatalf("request %d was refused: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := a.Check(p, "free", 1); !errors.Is(err, ErrRateLimited) {
|
||||||
|
t.Errorf("the fourth request was allowed: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTokenLimitsUseTheEstimateAndThenSettle(t *testing.T) {
|
||||||
|
// The real token count only exists once the request has run, so the check
|
||||||
|
// has to work on an estimate. A caller that consistently underestimates
|
||||||
|
// must still end up bounded.
|
||||||
|
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||||
|
p := principal("k1", func(e *store.Effective) { e.TPM = intp(1000) })
|
||||||
|
|
||||||
|
// Claims 10 tokens, actually uses 900.
|
||||||
|
res, err := a.Check(p, "free", 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 400, CompletionTokens: 500})
|
||||||
|
|
||||||
|
// The settled figure is what the next check sees, so a second request of
|
||||||
|
// the same shape cannot fit.
|
||||||
|
if _, err := a.Check(p, "free", 200); !errors.Is(err, ErrTokenLimited) {
|
||||||
|
t.Errorf("the underestimate was never settled: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoLimitIsNotALimitOfZero(t *testing.T) {
|
||||||
|
// The distinction the store keeps with a nil pointer. Getting it wrong here
|
||||||
|
// would silently forbid every request for every key with no limits set.
|
||||||
|
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||||
|
p := principal("k1", nil)
|
||||||
|
|
||||||
|
for range 50 {
|
||||||
|
if _, err := a.Check(p, "paid", 10_000); err != nil {
|
||||||
|
t.Fatalf("a key with no limits was refused: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnUnauthenticatedRequestIsNotAccounted(t *testing.T) {
|
||||||
|
// With auth off there is no key to charge, and Check must not refuse.
|
||||||
|
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||||
|
res, err := a.Check(auth.Principal{}, "paid", 1000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a.Record(res, store.Usage{PromptTokens: 1000})
|
||||||
|
if spent, _, _ := a.Status(auth.Principal{}); spent != 0 {
|
||||||
|
t.Errorf("an anonymous request was charged %d", spent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBudgetSurvivesARestart(t *testing.T) {
|
||||||
|
// The counters live in memory; the store is what makes them survive.
|
||||||
|
ctx := context.Background()
|
||||||
|
path := filepath.Join(t.TempDir(), "acct.db")
|
||||||
|
s, err := store.OpenSQLite(ctx, path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
must(t, s.PutOrg(ctx, store.Org{ID: "o", Name: "o"}))
|
||||||
|
must(t, s.PutTeam(ctx, store.Team{ID: "t", OrgID: "o", Name: "t"}))
|
||||||
|
must(t, s.PutProject(ctx, store.Project{ID: "proj", TeamID: "t", Name: "p"}))
|
||||||
|
_, k, _ := store.NewKey("k1", "n", "proj")
|
||||||
|
k.CreatedAt = time.Now()
|
||||||
|
must(t, s.CreateKey(ctx, k))
|
||||||
|
|
||||||
|
must(t, s.WriteUsage(ctx, []store.Usage{
|
||||||
|
{At: time.Now(), KeyID: "k1", ProjectID: "proj", Model: "paid",
|
||||||
|
ServedModel: "paid", CostMicros: 5000, Status: 200},
|
||||||
|
}))
|
||||||
|
|
||||||
|
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||||
|
must(t, a.RestoreBudgets(ctx, s, store.PeriodMonth))
|
||||||
|
|
||||||
|
p := principal("k1", func(e *store.Effective) {
|
||||||
|
e.Budget = i64p(5000)
|
||||||
|
e.BudgetPeriod = store.PeriodMonth
|
||||||
|
})
|
||||||
|
if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) {
|
||||||
|
t.Errorf("the budget was reset by the restart: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBudgetsResetAtThePeriodBoundary(t *testing.T) {
|
||||||
|
b := NewBudgets()
|
||||||
|
now := time.Date(2026, 1, 31, 23, 59, 0, 0, time.UTC)
|
||||||
|
b.now = func() time.Time { return now }
|
||||||
|
|
||||||
|
b.Add("k1", store.PeriodMonth, 900)
|
||||||
|
if got := b.Spent("k1", store.PeriodMonth); got != 900 {
|
||||||
|
t.Fatalf("spent %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
now = now.Add(2 * time.Minute) // into February
|
||||||
|
if got := b.Spent("k1", store.PeriodMonth); got != 0 {
|
||||||
|
t.Errorf("the counter carried %d across the month boundary", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDailyAndMonthlyBoundariesDiffer(t *testing.T) {
|
||||||
|
b := NewBudgets()
|
||||||
|
now := time.Date(2026, 3, 15, 12, 0, 0, 0, time.UTC)
|
||||||
|
b.now = func() time.Time { return now }
|
||||||
|
|
||||||
|
b.Add("daily", store.PeriodDay, 100)
|
||||||
|
b.Add("monthly", store.PeriodMonth, 100)
|
||||||
|
|
||||||
|
now = now.Add(24 * time.Hour)
|
||||||
|
if got := b.Spent("daily", store.PeriodDay); got != 0 {
|
||||||
|
t.Errorf("the daily counter carried %d across a day", got)
|
||||||
|
}
|
||||||
|
if got := b.Spent("monthly", store.PeriodMonth); got != 100 {
|
||||||
|
t.Errorf("the monthly counter lost %d within a month", 100-got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheWindowDoesNotAllowTwiceTheLimit(t *testing.T) {
|
||||||
|
// The reason for a sliding window. With a fixed one, a caller sends its
|
||||||
|
// whole allowance at the end of a window and again at the start of the
|
||||||
|
// next, so the observed rate is double the limit exactly when the system is
|
||||||
|
// least able to take it.
|
||||||
|
l := NewLimiter()
|
||||||
|
base := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||||
|
now := base.Add(59 * time.Second)
|
||||||
|
l.now = func() time.Time { return now }
|
||||||
|
|
||||||
|
const limit = 60
|
||||||
|
sent := 0
|
||||||
|
for range limit {
|
||||||
|
if l.Allow("k", limit, 1) {
|
||||||
|
sent++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if sent != limit {
|
||||||
|
t.Fatalf("the first burst only managed %d", sent)
|
||||||
|
}
|
||||||
|
|
||||||
|
// One second later, in the next fixed window.
|
||||||
|
now = base.Add(61 * time.Second)
|
||||||
|
second := 0
|
||||||
|
for range limit {
|
||||||
|
if l.Allow("k", limit, 1) {
|
||||||
|
second++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Some allowance returns as the old window slides out of view, but nowhere
|
||||||
|
// near the whole of it.
|
||||||
|
if second > limit/2 {
|
||||||
|
t.Errorf("a second burst of %d passed immediately after %d; the window is not sliding",
|
||||||
|
second, sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLimiterIsSafeUnderConcurrency(t *testing.T) {
|
||||||
|
// Checking and charging have to be one operation, or two concurrent
|
||||||
|
// requests both see room and both take it.
|
||||||
|
l := NewLimiter()
|
||||||
|
const limit = 100
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
var mu sync.Mutex
|
||||||
|
allowed := 0
|
||||||
|
for range 500 {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
if l.Allow("k", limit, 1) {
|
||||||
|
mu.Lock()
|
||||||
|
allowed++
|
||||||
|
mu.Unlock()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
if allowed != limit {
|
||||||
|
t.Errorf("%d requests passed a limit of %d", allowed, limit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func must(t *testing.T, err error) {
|
||||||
|
t.Helper()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
package accounting
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PeriodStart is when the current budget period began.
|
||||||
|
//
|
||||||
|
// Periods are anchored to UTC rather than to a per-key start date, so every
|
||||||
|
// key's day and month boundary is the same instant. A key whose period started
|
||||||
|
// when it was created would make "spend this month" mean something different
|
||||||
|
// for every key, and make the restore query unanswerable in one pass.
|
||||||
|
func PeriodStart(p store.Period, now time.Time) time.Time {
|
||||||
|
u := now.UTC()
|
||||||
|
switch p {
|
||||||
|
case store.PeriodDay:
|
||||||
|
return time.Date(u.Year(), u.Month(), u.Day(), 0, 0, 0, 0, time.UTC)
|
||||||
|
case store.PeriodMonth:
|
||||||
|
return time.Date(u.Year(), u.Month(), 1, 0, 0, 0, 0, time.UTC)
|
||||||
|
default:
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Budgets tracks spend per key in memory, so a budget check costs no query.
|
||||||
|
//
|
||||||
|
// The database catches up through the batch writer. That means the in-memory
|
||||||
|
// figure leads the stored one, which is the right direction: a budget that only
|
||||||
|
// counted what had already been written would let a burst overspend by however
|
||||||
|
// much the writer was behind.
|
||||||
|
type Budgets struct {
|
||||||
|
now func() time.Time
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
spend map[string]counter
|
||||||
|
}
|
||||||
|
|
||||||
|
type counter struct {
|
||||||
|
micros int64
|
||||||
|
since time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewBudgets builds an empty tracker.
|
||||||
|
func NewBudgets() *Budgets {
|
||||||
|
return &Budgets{now: time.Now, spend: map[string]counter{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Restore seeds the counters from what the store already recorded.
|
||||||
|
//
|
||||||
|
// A restart must not reset a budget, which it would if the counter lived only
|
||||||
|
// in memory. The caller passes the spend since each period's start.
|
||||||
|
func (b *Budgets) Restore(period store.Period, spend map[string]int64) {
|
||||||
|
start := PeriodStart(period, b.now())
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
for key, micros := range spend {
|
||||||
|
b.spend[key] = counter{micros: micros, since: start}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Spent returns a key's spend in the current period, resetting it if the period
|
||||||
|
// has rolled over.
|
||||||
|
func (b *Budgets) Spent(keyID string, period store.Period) int64 {
|
||||||
|
start := PeriodStart(period, b.now())
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
return b.currentLocked(keyID, start).micros
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add records spend.
|
||||||
|
func (b *Budgets) Add(keyID string, period store.Period, micros int64) {
|
||||||
|
if micros == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
start := PeriodStart(period, b.now())
|
||||||
|
b.mu.Lock()
|
||||||
|
defer b.mu.Unlock()
|
||||||
|
c := b.currentLocked(keyID, start)
|
||||||
|
c.micros += micros
|
||||||
|
b.spend[keyID] = c
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exhausted reports whether a key has spent its budget.
|
||||||
|
func (b *Budgets) Exhausted(keyID string, period store.Period, budget int64) bool {
|
||||||
|
return b.Spent(keyID, period) >= budget
|
||||||
|
}
|
||||||
|
|
||||||
|
// currentLocked returns the counter, rolled over if the period changed.
|
||||||
|
func (b *Budgets) currentLocked(keyID string, start time.Time) counter {
|
||||||
|
c, ok := b.spend[keyID]
|
||||||
|
if !ok || c.since.Before(start) {
|
||||||
|
// Either never seen, or the period rolled over while the process was
|
||||||
|
// running. Both start from zero.
|
||||||
|
c = counter{since: start}
|
||||||
|
b.spend[keyID] = c
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
// Package accounting turns finished requests into spend, enforces what a key
|
||||||
|
// is allowed, and gets both into the store without touching the request path
|
||||||
|
// (plan §7.9.3, §7.9.4).
|
||||||
|
package accounting
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Price is what one model costs, in micro-units per thousand tokens.
|
||||||
|
//
|
||||||
|
// Micro-units rather than a float because spend is accumulated by repeated
|
||||||
|
// addition, and repeated addition of floats drifts. What a unit means is the
|
||||||
|
// operator's: self-hosted models have no per-token price, so it may be
|
||||||
|
// GPU-seconds, a notional rate, or 1000 everywhere — which turns a budget into
|
||||||
|
// a token quota.
|
||||||
|
type Price struct {
|
||||||
|
PromptPer1k int64 `koanf:"prompt_per_1k"`
|
||||||
|
CompletionPer1k int64 `koanf:"completion_per_1k"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Zero reports whether this model spends nothing.
|
||||||
|
//
|
||||||
|
// It matters beyond arithmetic: a zero-cost model stays available to a key
|
||||||
|
// whose budget is spent, which is what makes the plan's budget fallback work
|
||||||
|
// (§7.7).
|
||||||
|
func (p Price) Zero() bool { return p.PromptPer1k == 0 && p.CompletionPer1k == 0 }
|
||||||
|
|
||||||
|
// Cost computes what a request spent, rounding up.
|
||||||
|
//
|
||||||
|
// Up rather than to nearest: rounding down would make a stream of tiny requests
|
||||||
|
// free, which is exactly the shape of traffic a quota exists to bound.
|
||||||
|
func (p Price) Cost(promptTokens, completionTokens int64) int64 {
|
||||||
|
return divUp(promptTokens*p.PromptPer1k, 1000) + divUp(completionTokens*p.CompletionPer1k, 1000)
|
||||||
|
}
|
||||||
|
|
||||||
|
func divUp(n, d int64) int64 {
|
||||||
|
if n <= 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return (n + d - 1) / d
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pricing resolves a model name to its price.
|
||||||
|
type Pricing struct {
|
||||||
|
mu sync.RWMutex
|
||||||
|
byModel map[string]Price
|
||||||
|
fallback Price
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewPricing builds a price table. Names are matched case-insensitively,
|
||||||
|
// because a model name reaches us from whatever the client typed.
|
||||||
|
func NewPricing(byModel map[string]Price, fallback Price) *Pricing {
|
||||||
|
table := make(map[string]Price, len(byModel))
|
||||||
|
for name, p := range byModel {
|
||||||
|
table[strings.ToLower(name)] = p
|
||||||
|
}
|
||||||
|
return &Pricing{byModel: table, fallback: fallback}
|
||||||
|
}
|
||||||
|
|
||||||
|
// For returns a model's price, or the fallback.
|
||||||
|
func (p *Pricing) For(model string) Price {
|
||||||
|
p.mu.RLock()
|
||||||
|
defer p.mu.RUnlock()
|
||||||
|
if v, ok := p.byModel[strings.ToLower(model)]; ok {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
return p.fallback
|
||||||
|
}
|
||||||
@@ -0,0 +1,130 @@
|
|||||||
|
package accounting
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Limiter enforces per-key rate limits.
|
||||||
|
//
|
||||||
|
// A sliding window rather than a fixed one. A fixed window lets a caller send
|
||||||
|
// its whole minute's allowance in the last second of one window and again in
|
||||||
|
// the first second of the next, so the observed rate is twice the configured
|
||||||
|
// limit exactly when the system is least able to take it.
|
||||||
|
//
|
||||||
|
// The window is approximated from two buckets, which is the usual compromise:
|
||||||
|
// exact enough that the doubling cannot happen, and bounded memory per key
|
||||||
|
// rather than a timestamp per request.
|
||||||
|
type Limiter struct {
|
||||||
|
now func() time.Time
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
keys map[string]*window
|
||||||
|
}
|
||||||
|
|
||||||
|
type window struct {
|
||||||
|
// current and previous are counts in this minute and the one before.
|
||||||
|
current, previous int64
|
||||||
|
start time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLimiter builds an empty limiter.
|
||||||
|
func NewLimiter() *Limiter {
|
||||||
|
return &Limiter{now: time.Now, keys: map[string]*window{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
const windowLen = time.Minute
|
||||||
|
|
||||||
|
// Allow charges n against a key's limit and reports whether it fits.
|
||||||
|
//
|
||||||
|
// Charging and checking are one operation on purpose: doing them separately
|
||||||
|
// lets two concurrent requests both see room and both take it.
|
||||||
|
func (l *Limiter) Allow(key string, limit int, n int64) bool {
|
||||||
|
if limit <= 0 {
|
||||||
|
// No limit configured. Zero would be a limit of zero, which is why
|
||||||
|
// callers pass a nil pointer rather than 0 for "unset".
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
now := l.now()
|
||||||
|
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
|
||||||
|
w := l.windowLocked(key, now)
|
||||||
|
if w.rate(now)+float64(n) > float64(limit) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
w.current += n
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Charge records usage that was already allowed, for reconciliation.
|
||||||
|
//
|
||||||
|
// Token limits are checked against an estimate before the request runs, because
|
||||||
|
// the real figure only exists once it has. The difference is charged here, so a
|
||||||
|
// caller whose requests are consistently larger than estimated is still bounded
|
||||||
|
// — one window later than it would be with a perfect estimate.
|
||||||
|
func (l *Limiter) Charge(key string, n int64) {
|
||||||
|
if n == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := l.now()
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
w := l.windowLocked(key, now)
|
||||||
|
w.current += n
|
||||||
|
if w.current < 0 {
|
||||||
|
w.current = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rate returns a key's current observed rate.
|
||||||
|
func (l *Limiter) Rate(key string) float64 {
|
||||||
|
now := l.now()
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
return l.windowLocked(key, now).rate(now)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Limiter) windowLocked(key string, now time.Time) *window {
|
||||||
|
w, ok := l.keys[key]
|
||||||
|
if !ok {
|
||||||
|
w = &window{start: now.Truncate(windowLen)}
|
||||||
|
l.keys[key] = w
|
||||||
|
}
|
||||||
|
w.roll(now)
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
// roll advances the window to now.
|
||||||
|
func (w *window) roll(now time.Time) {
|
||||||
|
start := now.Truncate(windowLen)
|
||||||
|
switch {
|
||||||
|
case start.Equal(w.start):
|
||||||
|
return
|
||||||
|
case start.Sub(w.start) == windowLen:
|
||||||
|
w.previous, w.current = w.current, 0
|
||||||
|
default:
|
||||||
|
// More than one window has passed with no traffic.
|
||||||
|
w.previous, w.current = 0, 0
|
||||||
|
}
|
||||||
|
w.start = start
|
||||||
|
}
|
||||||
|
|
||||||
|
// rate weights the previous window by how much of it is still in view.
|
||||||
|
func (w *window) rate(now time.Time) float64 {
|
||||||
|
elapsed := now.Sub(w.start)
|
||||||
|
if elapsed >= windowLen {
|
||||||
|
return float64(w.current)
|
||||||
|
}
|
||||||
|
remaining := 1 - float64(elapsed)/float64(windowLen)
|
||||||
|
return float64(w.previous)*remaining + float64(w.current)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Forget drops a key's window. Used when a key is revoked; keeping it would
|
||||||
|
// leak memory for every key that ever called.
|
||||||
|
func (l *Limiter) Forget(key string) {
|
||||||
|
l.mu.Lock()
|
||||||
|
delete(l.keys, key)
|
||||||
|
l.mu.Unlock()
|
||||||
|
}
|
||||||
@@ -0,0 +1,189 @@
|
|||||||
|
package accounting
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RecorderConfig tunes the batch writer.
|
||||||
|
type RecorderConfig struct {
|
||||||
|
// FlushInterval and BatchSize decide when a batch is written. Whichever
|
||||||
|
// comes first.
|
||||||
|
FlushInterval time.Duration
|
||||||
|
BatchSize int
|
||||||
|
// Capacity bounds the buffer. Beyond it, records are dropped rather than
|
||||||
|
// blocking a request — see Record.
|
||||||
|
Capacity int
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultRecorderConfig returns sensible values.
|
||||||
|
func DefaultRecorderConfig() RecorderConfig {
|
||||||
|
return RecorderConfig{FlushInterval: 200 * time.Millisecond, BatchSize: 500, Capacity: 8192}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *RecorderConfig) normalize() {
|
||||||
|
d := DefaultRecorderConfig()
|
||||||
|
if c.FlushInterval <= 0 {
|
||||||
|
c.FlushInterval = d.FlushInterval
|
||||||
|
}
|
||||||
|
if c.BatchSize <= 0 {
|
||||||
|
c.BatchSize = d.BatchSize
|
||||||
|
}
|
||||||
|
if c.Capacity <= 0 {
|
||||||
|
c.Capacity = d.Capacity
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Toucher is the part of the authenticator the recorder flushes alongside
|
||||||
|
// usage, so last-used timestamps ride the same transaction schedule.
|
||||||
|
type Toucher interface {
|
||||||
|
FlushTouches(ctx context.Context) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recorder batches usage records into the store.
|
||||||
|
//
|
||||||
|
// The buffered channel is the whole design. A request must never wait on a
|
||||||
|
// database write, so Record is a non-blocking send and the writer drains it on
|
||||||
|
// its own schedule.
|
||||||
|
type Recorder struct {
|
||||||
|
cfg RecorderConfig
|
||||||
|
store store.Store
|
||||||
|
touch Toucher
|
||||||
|
log *slog.Logger
|
||||||
|
|
||||||
|
ch chan store.Usage
|
||||||
|
done chan struct{}
|
||||||
|
wg sync.WaitGroup
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
dropped int64
|
||||||
|
written int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRecorder builds a recorder. Start begins writing.
|
||||||
|
func NewRecorder(s store.Store, touch Toucher, cfg RecorderConfig, log *slog.Logger) *Recorder {
|
||||||
|
cfg.normalize()
|
||||||
|
if log == nil {
|
||||||
|
log = slog.Default()
|
||||||
|
}
|
||||||
|
return &Recorder{
|
||||||
|
cfg: cfg, store: s, touch: touch, log: log,
|
||||||
|
ch: make(chan store.Usage, cfg.Capacity),
|
||||||
|
done: make(chan struct{}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Record queues one request's accounting.
|
||||||
|
//
|
||||||
|
// Never blocks. If the buffer is full — a database stalled long enough to fill
|
||||||
|
// eight thousand records — the record is dropped and counted. Losing some
|
||||||
|
// accounting is bad; making every request wait on a stalled database is worse,
|
||||||
|
// because it converts a reporting problem into an outage.
|
||||||
|
func (r *Recorder) Record(u store.Usage) {
|
||||||
|
select {
|
||||||
|
case r.ch <- u:
|
||||||
|
default:
|
||||||
|
r.mu.Lock()
|
||||||
|
r.dropped++
|
||||||
|
n := r.dropped
|
||||||
|
r.mu.Unlock()
|
||||||
|
if n == 1 || n%1000 == 0 {
|
||||||
|
r.log.Warn("dropping usage records: the store is not keeping up",
|
||||||
|
"dropped_total", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start runs the writer until ctx ends.
|
||||||
|
func (r *Recorder) Start(ctx context.Context) {
|
||||||
|
r.wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer r.wg.Done()
|
||||||
|
defer close(r.done)
|
||||||
|
r.run(ctx)
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop waits for the writer to drain and finish.
|
||||||
|
func (r *Recorder) Stop() { r.wg.Wait() }
|
||||||
|
|
||||||
|
func (r *Recorder) run(ctx context.Context) {
|
||||||
|
t := time.NewTicker(r.cfg.FlushInterval)
|
||||||
|
defer t.Stop()
|
||||||
|
|
||||||
|
batch := make([]store.Usage, 0, r.cfg.BatchSize)
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
// Drain what is already queued before leaving. These are completed
|
||||||
|
// requests that have already been served; throwing them away would
|
||||||
|
// mean a clean shutdown silently loses the last moments of billing.
|
||||||
|
r.drain(&batch)
|
||||||
|
r.flush(context.WithoutCancel(ctx), batch)
|
||||||
|
return
|
||||||
|
|
||||||
|
case u := <-r.ch:
|
||||||
|
batch = append(batch, u)
|
||||||
|
if len(batch) >= r.cfg.BatchSize {
|
||||||
|
r.flush(ctx, batch)
|
||||||
|
batch = batch[:0]
|
||||||
|
}
|
||||||
|
|
||||||
|
case <-t.C:
|
||||||
|
r.drain(&batch)
|
||||||
|
if len(batch) > 0 {
|
||||||
|
r.flush(ctx, batch)
|
||||||
|
batch = batch[:0]
|
||||||
|
}
|
||||||
|
if err := r.touchFlush(ctx); err != nil {
|
||||||
|
r.log.Warn("failed to write last-used timestamps", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// drain moves everything queued into the batch without blocking.
|
||||||
|
func (r *Recorder) drain(batch *[]store.Usage) {
|
||||||
|
for len(*batch) < r.cfg.BatchSize {
|
||||||
|
select {
|
||||||
|
case u := <-r.ch:
|
||||||
|
*batch = append(*batch, u)
|
||||||
|
default:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Recorder) flush(ctx context.Context, batch []store.Usage) {
|
||||||
|
if len(batch) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := r.store.WriteUsage(ctx, batch); err != nil {
|
||||||
|
// The records are gone either way; what matters is that this is loud.
|
||||||
|
// Silently losing billing data is the kind of thing discovered at the
|
||||||
|
// end of a month.
|
||||||
|
r.log.Error("failed to write usage records", "count", len(batch), "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.mu.Lock()
|
||||||
|
r.written += int64(len(batch))
|
||||||
|
r.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Recorder) touchFlush(ctx context.Context) error {
|
||||||
|
if r.touch == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return r.touch.FlushTouches(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stats reports what the recorder has done, for metrics and tests.
|
||||||
|
func (r *Recorder) Stats() (written, dropped int64) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
return r.written, r.dropped
|
||||||
|
}
|
||||||
@@ -0,0 +1,210 @@
|
|||||||
|
package accounting
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeStore records what was written and can be made slow or broken.
|
||||||
|
type fakeStore struct {
|
||||||
|
store.Store
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
batches [][]store.Usage
|
||||||
|
|
||||||
|
block chan struct{}
|
||||||
|
fail atomic.Bool
|
||||||
|
touched atomic.Int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeStore) WriteUsage(_ context.Context, rows []store.Usage) error {
|
||||||
|
if f.block != nil {
|
||||||
|
<-f.block
|
||||||
|
}
|
||||||
|
if f.fail.Load() {
|
||||||
|
return errors.New("store is down")
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.batches = append(f.batches, append([]store.Usage(nil), rows...))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeStore) rows() []store.Usage {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
var out []store.Usage
|
||||||
|
for _, b := range f.batches {
|
||||||
|
out = append(out, b...)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeStore) batchCount() int {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return len(f.batches)
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeToucher struct{ n *atomic.Int64 }
|
||||||
|
|
||||||
|
func (f fakeToucher) FlushTouches(context.Context) error {
|
||||||
|
f.n.Add(1)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecordsAreBatchedNotWrittenOneByOne(t *testing.T) {
|
||||||
|
// The whole point of the buffer: a hundred requests must not be a hundred
|
||||||
|
// transactions.
|
||||||
|
f := &fakeStore{}
|
||||||
|
r := NewRecorder(f, nil, RecorderConfig{FlushInterval: 20 * time.Millisecond, BatchSize: 500},
|
||||||
|
slog.New(slog.DiscardHandler))
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
r.Start(ctx)
|
||||||
|
|
||||||
|
for i := range 100 {
|
||||||
|
r.Record(store.Usage{KeyID: "k", PromptTokens: int64(i)})
|
||||||
|
}
|
||||||
|
waitFor(t, time.Second, func() bool { return len(f.rows()) == 100 })
|
||||||
|
|
||||||
|
if n := f.batchCount(); n > 5 {
|
||||||
|
t.Errorf("100 records took %d transactions", n)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
r.Stop()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecordNeverBlocksARequest(t *testing.T) {
|
||||||
|
// A stalled database must not become an outage. Losing accounting is bad;
|
||||||
|
// making every request wait on the store is worse.
|
||||||
|
f := &fakeStore{block: make(chan struct{})}
|
||||||
|
r := NewRecorder(f, nil, RecorderConfig{
|
||||||
|
FlushInterval: time.Millisecond, BatchSize: 1, Capacity: 8,
|
||||||
|
}, slog.New(slog.DiscardHandler))
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
r.Start(ctx)
|
||||||
|
defer func() { close(f.block); cancel(); r.Stop() }()
|
||||||
|
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
for range 1000 {
|
||||||
|
r.Record(store.Usage{KeyID: "k"})
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("Record blocked while the store was stalled")
|
||||||
|
}
|
||||||
|
|
||||||
|
_, dropped := r.Stats()
|
||||||
|
if dropped == 0 {
|
||||||
|
t.Error("the overflow was not counted, so nobody would ever know")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAShutdownDoesNotLoseWhatIsAlreadyQueued(t *testing.T) {
|
||||||
|
// Those are completed requests that were already served. Throwing them away
|
||||||
|
// means a clean shutdown silently loses the last moments of billing.
|
||||||
|
f := &fakeStore{}
|
||||||
|
r := NewRecorder(f, nil, RecorderConfig{FlushInterval: time.Hour, BatchSize: 10_000},
|
||||||
|
slog.New(slog.DiscardHandler))
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
r.Start(ctx)
|
||||||
|
|
||||||
|
for range 50 {
|
||||||
|
r.Record(store.Usage{KeyID: "k"})
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
r.Stop()
|
||||||
|
|
||||||
|
if got := len(f.rows()); got != 50 {
|
||||||
|
t.Errorf("a clean shutdown wrote %d of 50 queued records", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFailedWriteIsLoudRatherThanSilent(t *testing.T) {
|
||||||
|
// Silently losing billing data is the kind of thing discovered at the end
|
||||||
|
// of a month.
|
||||||
|
f := &fakeStore{}
|
||||||
|
f.fail.Store(true)
|
||||||
|
logged := &countingHandler{}
|
||||||
|
r := NewRecorder(f, nil, RecorderConfig{FlushInterval: 10 * time.Millisecond, BatchSize: 1},
|
||||||
|
slog.New(logged))
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
r.Start(ctx)
|
||||||
|
r.Record(store.Usage{KeyID: "k"})
|
||||||
|
|
||||||
|
waitFor(t, time.Second, func() bool { return logged.count(slog.LevelError) > 0 })
|
||||||
|
cancel()
|
||||||
|
r.Stop()
|
||||||
|
|
||||||
|
if written, _ := r.Stats(); written != 0 {
|
||||||
|
t.Errorf("a failed write was counted as %d written", written)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLastUsedRidesTheSameSchedule(t *testing.T) {
|
||||||
|
var n atomic.Int64
|
||||||
|
f := &fakeStore{}
|
||||||
|
r := NewRecorder(f, fakeToucher{n: &n}, RecorderConfig{FlushInterval: 10 * time.Millisecond},
|
||||||
|
slog.New(slog.DiscardHandler))
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
r.Start(ctx)
|
||||||
|
waitFor(t, time.Second, func() bool { return n.Load() > 0 })
|
||||||
|
cancel()
|
||||||
|
r.Stop()
|
||||||
|
}
|
||||||
|
|
||||||
|
type countingHandler struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
levels []slog.Level
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *countingHandler) Enabled(context.Context, slog.Level) bool { return true }
|
||||||
|
func (c *countingHandler) WithAttrs([]slog.Attr) slog.Handler { return c }
|
||||||
|
func (c *countingHandler) WithGroup(string) slog.Handler { return c }
|
||||||
|
func (c *countingHandler) Handle(_ context.Context, r slog.Record) error {
|
||||||
|
c.mu.Lock()
|
||||||
|
c.levels = append(c.levels, r.Level)
|
||||||
|
c.mu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *countingHandler) count(l slog.Level) int {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
n := 0
|
||||||
|
for _, got := range c.levels {
|
||||||
|
if got == l {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitFor(t *testing.T, limit time.Duration, pred func() bool) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(limit)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if pred() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatal("condition never became true")
|
||||||
|
}
|
||||||
@@ -0,0 +1,899 @@
|
|||||||
|
// Package agent is the node side of the control plane stream (plan §5.6).
|
||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"crypto/tls"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"math/rand/v2"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/net/http2"
|
||||||
|
|
||||||
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Config configures the agent.
|
||||||
|
type Config struct {
|
||||||
|
NodeID string `koanf:"node_id"`
|
||||||
|
Server string `koanf:"server"`
|
||||||
|
Hostname string `koanf:"hostname"`
|
||||||
|
MgmtAddr string `koanf:"mgmt_addr"`
|
||||||
|
Labels map[string]string `koanf:"labels"`
|
||||||
|
|
||||||
|
// MgmtInterface is the ethernet interface NCCL bootstraps over.
|
||||||
|
MgmtInterface string `koanf:"mgmt_interface"`
|
||||||
|
// Fabrics labels this node's HCAs. It is explicit on purpose: OpenSM's
|
||||||
|
// default subnet prefix is the same on every fabric, so nothing can be
|
||||||
|
// inferred from the wire (plan §5.2).
|
||||||
|
Fabrics []inventory.FabricAttachment `koanf:"fabrics"`
|
||||||
|
|
||||||
|
HeartbeatInterval time.Duration `koanf:"heartbeat_interval"`
|
||||||
|
// ReconnectBase is the first reconnect delay; it backs off from there.
|
||||||
|
ReconnectBase time.Duration `koanf:"reconnect_base"`
|
||||||
|
ReconnectMax time.Duration `koanf:"reconnect_max"`
|
||||||
|
|
||||||
|
// AdoptReadyTimeout bounds how long an adopted engine is given to answer a
|
||||||
|
// health check before the agent stops waiting on it.
|
||||||
|
//
|
||||||
|
// Shorter than a startup timeout on purpose: this engine is already
|
||||||
|
// running, so it is not loading a model, it is either serving or it is not.
|
||||||
|
AdoptReadyTimeout time.Duration `koanf:"adopt_ready_timeout"`
|
||||||
|
|
||||||
|
// TLS authenticates both ends of the control-plane stream. Leaving it unset
|
||||||
|
// keeps the plaintext behaviour, which only belongs on a trusted network.
|
||||||
|
TLS tlsconf.Config `koanf:"tls"`
|
||||||
|
|
||||||
|
// ModelDirs are the directories this node keeps model weights in.
|
||||||
|
//
|
||||||
|
// Empty by default, and empty means the node is not asked to look — which
|
||||||
|
// is not the same as having nothing. Storage differs at every installation,
|
||||||
|
// so llmgw observes it rather than owning it: a Lustre mount, an image
|
||||||
|
// exported over the fabric, local NVMe and a copy made by hand all look
|
||||||
|
// identical from here, and placement only needs to know whether the file
|
||||||
|
// is present.
|
||||||
|
ModelDirs []string `koanf:"model_dirs"`
|
||||||
|
|
||||||
|
// WeightsCache is where a remote source is downloaded to. Defaults to the
|
||||||
|
// first entry of ModelDirs, because a node that says where its models live
|
||||||
|
// has already answered this question.
|
||||||
|
//
|
||||||
|
// Fetching is the simple arrangement: every node keeps its own copy. The
|
||||||
|
// exotic ones need no code — weights on a shared filer or an image exported
|
||||||
|
// over the fabric are simply present, and present means nothing is fetched.
|
||||||
|
WeightsCache string `koanf:"weights_cache"`
|
||||||
|
|
||||||
|
Discovery discovery.Options `koanf:"discovery"`
|
||||||
|
Engines engines.Config `koanf:"engines"`
|
||||||
|
Superv supervisor.Config `koanf:"supervisor"`
|
||||||
|
|
||||||
|
Version string
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultConfig returns sensible defaults.
|
||||||
|
func DefaultConfig() Config {
|
||||||
|
return Config{
|
||||||
|
MgmtInterface: "eth0",
|
||||||
|
HeartbeatInterval: 5 * time.Second,
|
||||||
|
ReconnectBase: time.Second,
|
||||||
|
ReconnectMax: 30 * time.Second,
|
||||||
|
AdoptReadyTimeout: 60 * time.Second,
|
||||||
|
Superv: supervisor.DefaultConfig(),
|
||||||
|
// NVML gives NVLink topology, so prefer it when this build has it.
|
||||||
|
Discovery: discovery.Options{PreferNVML: true},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) normalize() error {
|
||||||
|
d := DefaultConfig()
|
||||||
|
if c.NodeID == "" {
|
||||||
|
return errors.New("node_id is required")
|
||||||
|
}
|
||||||
|
if c.Server == "" {
|
||||||
|
return errors.New("server is required")
|
||||||
|
}
|
||||||
|
if err := c.TLS.Validate(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := c.normalizeServerURL(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if c.MgmtInterface == "" {
|
||||||
|
c.MgmtInterface = d.MgmtInterface
|
||||||
|
}
|
||||||
|
if c.HeartbeatInterval <= 0 {
|
||||||
|
c.HeartbeatInterval = d.HeartbeatInterval
|
||||||
|
}
|
||||||
|
if c.ReconnectBase <= 0 {
|
||||||
|
c.ReconnectBase = d.ReconnectBase
|
||||||
|
}
|
||||||
|
if c.AdoptReadyTimeout <= 0 {
|
||||||
|
c.AdoptReadyTimeout = d.AdoptReadyTimeout
|
||||||
|
}
|
||||||
|
if c.ReconnectMax < c.ReconnectBase {
|
||||||
|
c.ReconnectMax = max(c.ReconnectBase, d.ReconnectMax)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizeServerURL makes the configured address a URL connect can use, and
|
||||||
|
// refuses the combination that would quietly undo the certificates: a scheme
|
||||||
|
// that says plaintext while TLS is configured. Silently connecting anyway would
|
||||||
|
// mean a cluster believing it has mutual authentication when it has none.
|
||||||
|
func (c *Config) normalizeServerURL() error {
|
||||||
|
secure := c.TLS.Enabled()
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(c.Server, "https://"):
|
||||||
|
if !secure {
|
||||||
|
return errors.New("server is an https:// address but no tls certificates are configured")
|
||||||
|
}
|
||||||
|
case strings.HasPrefix(c.Server, "http://"):
|
||||||
|
if secure {
|
||||||
|
return errors.New("tls is configured but server is an http:// address; use https://")
|
||||||
|
}
|
||||||
|
case strings.Contains(c.Server, "://"):
|
||||||
|
return fmt.Errorf("server %q must be an http:// or https:// address", c.Server)
|
||||||
|
case secure:
|
||||||
|
c.Server = "https://" + c.Server
|
||||||
|
default:
|
||||||
|
c.Server = "http://" + c.Server
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Agent runs on a GPU node.
|
||||||
|
type Agent struct {
|
||||||
|
cfg Config
|
||||||
|
log *slog.Logger
|
||||||
|
sup *supervisor.Supervisor
|
||||||
|
reg *engines.Registry
|
||||||
|
http connectHTTPClient
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
inv discovery.Inventory
|
||||||
|
// shardReplica remembers which replica a shard belongs to, so a status
|
||||||
|
// report can name both. The control plane keys on the replica.
|
||||||
|
shardReplica map[string]string
|
||||||
|
fetcher *weights.Fetcher
|
||||||
|
|
||||||
|
// lastWeights is the fingerprint of the file list most recently sent, so a
|
||||||
|
// heartbeat carries the list only when it changed. Touched only by the
|
||||||
|
// heartbeat goroutine, which is the only thing that reads or writes it.
|
||||||
|
lastWeights string
|
||||||
|
}
|
||||||
|
|
||||||
|
// connectHTTPClient is the HTTP client connect uses; an interface so tests can
|
||||||
|
// hand in an httptest server's client.
|
||||||
|
type connectHTTPClient interface {
|
||||||
|
Do(*http.Request) (*http.Response, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// New builds an agent.
|
||||||
|
func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, error) {
|
||||||
|
if err := cfg.normalize(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if log == nil {
|
||||||
|
log = slog.Default()
|
||||||
|
}
|
||||||
|
if httpClient == nil {
|
||||||
|
httpClient = http.DefaultClient
|
||||||
|
}
|
||||||
|
|
||||||
|
health := supervisor.NewHTTPHealth(3 * time.Second)
|
||||||
|
sup, err := supervisor.New(cfg.Superv, health, log)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &Agent{
|
||||||
|
cfg: cfg, log: log, sup: sup,
|
||||||
|
reg: engines.NewRegistry(cfg.Engines),
|
||||||
|
http: httpClient,
|
||||||
|
shardReplica: map[string]string{},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Supervisor exposes the process supervisor, for tests and `ctl logs`.
|
||||||
|
func (a *Agent) Supervisor() *supervisor.Supervisor { return a.sup }
|
||||||
|
|
||||||
|
// Run connects to the control plane and stays connected until ctx ends.
|
||||||
|
//
|
||||||
|
// Reconnecting forever is the right behaviour for a node agent: a control plane
|
||||||
|
// restart or a network partition must not require touching every GPU node, and
|
||||||
|
// the engines keep serving traffic throughout because the router talks to them
|
||||||
|
// directly.
|
||||||
|
func (a *Agent) Run(ctx context.Context) error {
|
||||||
|
if err := a.scan(ctx); err != nil {
|
||||||
|
// Not fatal: a node whose GPUs cannot be read should still register, so
|
||||||
|
// an operator can see it and why it is useless.
|
||||||
|
a.log.Error("hardware discovery failed", "err", err)
|
||||||
|
}
|
||||||
|
a.recoverOrphans()
|
||||||
|
|
||||||
|
attempt := 0
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
err := a.session(ctx)
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
attempt++
|
||||||
|
delay := a.reconnectDelay(attempt)
|
||||||
|
a.log.Warn("disconnected from the control plane, retrying",
|
||||||
|
"err", err, "in", delay.Round(time.Millisecond), "attempt", attempt)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
case <-time.After(delay):
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
attempt = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Engines keep running when the agent stops on purpose: an agent restart
|
||||||
|
// must not interrupt served traffic. Orphan adoption picks them up again.
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Agent) reconnectDelay(attempt int) time.Duration {
|
||||||
|
d := a.cfg.ReconnectBase << min(attempt-1, 16)
|
||||||
|
if d > a.cfg.ReconnectMax || d <= 0 {
|
||||||
|
d = a.cfg.ReconnectMax
|
||||||
|
}
|
||||||
|
// Jitter, so a fleet does not reconnect in lockstep after a control plane
|
||||||
|
// restart and knock it over again.
|
||||||
|
return d/2 + time.Duration(rand.Int64N(int64(d/2))) //nolint:gosec // jitter
|
||||||
|
}
|
||||||
|
|
||||||
|
// recoverOrphans takes back engines a previous agent left running.
|
||||||
|
//
|
||||||
|
// Engines deliberately outlive their agent — the router talks to them directly,
|
||||||
|
// so an agent restart must not interrupt served traffic. The cost is that they
|
||||||
|
// have to be found again, and anything that cannot be identified has to be
|
||||||
|
// killed: a process holding GPUs that nothing can describe is worse than no
|
||||||
|
// process at all, because placement will hand those GPUs out again.
|
||||||
|
func (a *Agent) recoverOrphans() {
|
||||||
|
orphans, err := a.sup.FindOrphans()
|
||||||
|
if err != nil {
|
||||||
|
a.log.Warn("scanning for leftover engines failed", "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, o := range orphans {
|
||||||
|
switch {
|
||||||
|
case !o.Alive:
|
||||||
|
// The process is gone; only its state file remains.
|
||||||
|
if err := a.sup.KillOrphan(o); err != nil {
|
||||||
|
a.log.Warn("cleaning up a dead shard's state failed", "shard", o.ShardID, "err", err)
|
||||||
|
}
|
||||||
|
case o.ReplicaID == "":
|
||||||
|
// Alive, but its state file could not be read, so it cannot be
|
||||||
|
// described to the control plane or matched to a replica.
|
||||||
|
a.log.Warn("killing an unidentifiable engine", "shard", o.ShardID, "pid", o.PID)
|
||||||
|
if err := a.sup.KillOrphan(o); err != nil {
|
||||||
|
a.log.Warn("killing it failed", "shard", o.ShardID, "err", err)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
sh, err := a.sup.Adopt(o)
|
||||||
|
if err != nil {
|
||||||
|
a.log.Warn("adopting a shard failed", "shard", o.ShardID, "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
a.shardReplica[o.ShardID] = o.ReplicaID
|
||||||
|
a.mu.Unlock()
|
||||||
|
a.log.Info("adopted a running engine",
|
||||||
|
"shard", o.ShardID, "replica", o.ReplicaID, "pid", o.PID)
|
||||||
|
go a.settleAdopted(sh, o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// settleAdopted brings an adopted engine to Ready, or lets it fail.
|
||||||
|
//
|
||||||
|
// Adoption stopped at "the process is alive", which left the shard in Starting
|
||||||
|
// with nothing that would ever move it. The control plane then waited out the
|
||||||
|
// profile's startup timeout and killed a perfectly healthy engine so it could
|
||||||
|
// start an identical one — so an agent restart, which exists precisely so that
|
||||||
|
// engines are not interrupted, cost every model on the node a full reload.
|
||||||
|
//
|
||||||
|
// Measured on node05: the engine answered /health the whole time, and the
|
||||||
|
// replica sat in Starting for three minutes until the timeout reaped it.
|
||||||
|
//
|
||||||
|
// Run in the background because recovery happens before the agent connects, and
|
||||||
|
// a slow health check must not delay registration — the control plane needs to
|
||||||
|
// hear about these shards promptly whatever state they are in.
|
||||||
|
func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), a.cfg.AdoptReadyTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
eng, err := a.reg.Get(o.Spec.Profile.Engine)
|
||||||
|
if err != nil {
|
||||||
|
a.log.Warn("cannot judge an adopted engine; leaving it to the control plane",
|
||||||
|
"shard", o.ShardID, "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// The same two paths a freshly started shard takes: a rank that serves HTTP
|
||||||
|
// is health-checked, a worker is judged by staying alive.
|
||||||
|
if eng.ServesHTTP(o.Spec) {
|
||||||
|
base := fmt.Sprintf("http://127.0.0.1:%d", o.Spec.HTTPPort)
|
||||||
|
err = a.sup.WaitReady(ctx, sh, base, o.Spec.Profile.ServedModelName, a.cfg.AdoptReadyTimeout)
|
||||||
|
} else {
|
||||||
|
err = a.sup.WaitAlive(ctx, sh)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
// Not killed here. An adopted engine that will not answer is the
|
||||||
|
// control plane's decision to replace, and it has the timeout for it;
|
||||||
|
// killing it locally would race that.
|
||||||
|
a.log.Warn("an adopted engine did not become ready",
|
||||||
|
"shard", o.ShardID, "replica", o.ReplicaID, "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.log.Info("adopted engine is serving",
|
||||||
|
"shard", o.ShardID, "replica", o.ReplicaID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// usableFabrics reports the fabrics this node can actually carry traffic on.
|
||||||
|
//
|
||||||
|
// Loud about what it drops. A fabric silently missing from a registration looks
|
||||||
|
// identical to one never configured, and the operator who typed the adapter
|
||||||
|
// name is the only person who can fix it — so the reason is logged here, on the
|
||||||
|
// machine where both halves are visible, rather than inferred later from a
|
||||||
|
// replica that will not converge.
|
||||||
|
func (a *Agent) usableFabrics(inv discovery.Inventory) []inventory.FabricAttachment {
|
||||||
|
declared := map[string][]string{}
|
||||||
|
for _, f := range a.cfg.Fabrics {
|
||||||
|
declared[f.Fabric] = f.HCAs
|
||||||
|
}
|
||||||
|
|
||||||
|
var out []inventory.FabricAttachment
|
||||||
|
for _, c := range discovery.CheckFabrics(declared, inv.HCAs) {
|
||||||
|
for _, p := range c.Problems {
|
||||||
|
a.log.Error("a declared fabric adapter cannot carry traffic",
|
||||||
|
"fabric", c.Fabric, "problem", p)
|
||||||
|
}
|
||||||
|
if !c.OK() {
|
||||||
|
a.log.Error("fabric not advertised; no declared adapter on it can carry traffic",
|
||||||
|
"fabric", c.Fabric,
|
||||||
|
"consequence", "no multi-node replica will be placed here")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, inventory.FabricAttachment{
|
||||||
|
Fabric: c.Fabric, HCAs: c.Usable, RateGbps: c.RateGbps,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// weights builds the fetcher lazily, so an agent that never starts a shard
|
||||||
|
// never needs a cache directory.
|
||||||
|
func (a *Agent) weights() *weights.Fetcher {
|
||||||
|
a.mu.Lock()
|
||||||
|
defer a.mu.Unlock()
|
||||||
|
if a.fetcher != nil {
|
||||||
|
return a.fetcher
|
||||||
|
}
|
||||||
|
cache := a.cfg.WeightsCache
|
||||||
|
if cache == "" && len(a.cfg.ModelDirs) > 0 {
|
||||||
|
cache = a.cfg.ModelDirs[0]
|
||||||
|
}
|
||||||
|
a.fetcher = &weights.Fetcher{
|
||||||
|
CacheDir: cache,
|
||||||
|
Log: a.log,
|
||||||
|
OnProgress: func(p weights.Progress) {
|
||||||
|
// Logged rather than silent: a replica that is downloading a
|
||||||
|
// hundred gigabytes and one that is wedged look identical from the
|
||||||
|
// control plane, and the difference matters to whoever is waiting.
|
||||||
|
a.log.Info("fetching weights",
|
||||||
|
"source", p.Source, "percent", p.Percent(),
|
||||||
|
"bytes", p.BytesDone, "total", p.BytesTotal)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return a.fetcher
|
||||||
|
}
|
||||||
|
|
||||||
|
// runningShards describes what this agent is supervising, for registration.
|
||||||
|
func (a *Agent) runningShards() []*llmgwv1.RunningShard {
|
||||||
|
var out []*llmgwv1.RunningShard
|
||||||
|
for _, st := range a.sup.Statuses() {
|
||||||
|
if st.State.Terminal() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sh, err := a.sup.Get(st.ID)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
spec := sh.Spec
|
||||||
|
out = append(out, &llmgwv1.RunningShard{
|
||||||
|
ReplicaId: spec.ReplicaID,
|
||||||
|
ShardId: st.ID,
|
||||||
|
Rank: int32(spec.Rank),
|
||||||
|
Nnodes: int32(spec.NNodes),
|
||||||
|
GpuIndices: int32Slice(spec.GPUIndices),
|
||||||
|
Hcas: spec.HCAs,
|
||||||
|
HttpPort: int32(spec.HTTPPort),
|
||||||
|
State: agentsvc.StateToWire(st.State),
|
||||||
|
Model: spec.Profile.ServedModelName,
|
||||||
|
Profile: spec.Profile.Name,
|
||||||
|
Pool: spec.Pool,
|
||||||
|
Fabric: spec.Fabric,
|
||||||
|
LeaderAddr: a.cfg.MgmtAddr,
|
||||||
|
LeaderPort: int32(spec.LeaderPort),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func int32Slice(v []int) []int32 {
|
||||||
|
if len(v) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]int32, len(v))
|
||||||
|
for i, x := range v {
|
||||||
|
out[i] = int32(x)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// scan refreshes the node's hardware inventory.
|
||||||
|
func (a *Agent) scan(ctx context.Context) error {
|
||||||
|
// Carried from the agent's own configuration rather than duplicated into
|
||||||
|
// the discovery block, so an operator names the directories once.
|
||||||
|
opts := a.cfg.Discovery
|
||||||
|
opts.ModelDirs = a.cfg.ModelDirs
|
||||||
|
inv, err := discovery.Discover(ctx, opts, a.log)
|
||||||
|
a.mu.Lock()
|
||||||
|
a.inv = inv
|
||||||
|
a.mu.Unlock()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// session runs one connection until it breaks.
|
||||||
|
func (a *Agent) session(ctx context.Context) error {
|
||||||
|
client := llmgwv1connect.NewAgentServiceClient(a.http, a.cfg.Server)
|
||||||
|
|
||||||
|
// parent is kept because the derived context is cancelled from inside this
|
||||||
|
// function; asking the derived one whether we are shutting down would
|
||||||
|
// always say yes, and every stream error would read as a clean close.
|
||||||
|
parent := ctx
|
||||||
|
ctx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
stream := client.Connect(ctx)
|
||||||
|
defer func() { _ = stream.CloseRequest() }()
|
||||||
|
|
||||||
|
a.mu.Lock()
|
||||||
|
inv := a.inv
|
||||||
|
a.mu.Unlock()
|
||||||
|
|
||||||
|
// Declared fabric membership against what the node actually has. A fabric
|
||||||
|
// whose adapters are absent or have no active port is not advertised at
|
||||||
|
// all, so the control plane will not place a multi-node replica on it —
|
||||||
|
// rather than placing one that starts, passes health checks and hangs on
|
||||||
|
// its first all-reduce.
|
||||||
|
reg := agentsvc.RegisterFromInventory(
|
||||||
|
a.cfg.NodeID, a.cfg.Hostname, a.cfg.MgmtAddr, a.cfg.Version,
|
||||||
|
a.cfg.Labels, inv, a.usableFabrics(inv))
|
||||||
|
// Every registration reports what is already running, not just the first
|
||||||
|
// one: a reconnect may be a control plane restart, and it has to recover
|
||||||
|
// the same way (plan §6.5).
|
||||||
|
reg.RunningShards = a.runningShards()
|
||||||
|
|
||||||
|
if err := stream.Send(&llmgwv1.AgentMessage{
|
||||||
|
Msg: &llmgwv1.AgentMessage_Register{Register: reg},
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("registering: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Closing the response is what actually unblocks Receive below. Cancelling
|
||||||
|
// the context is not enough: HTTP/2 watches it while the request body is
|
||||||
|
// being written, and a bidirectional stream spends its life blocked on the
|
||||||
|
// read side instead. Without this an agent told to stop keeps reading until
|
||||||
|
// something kills it — which for systemd means the full stop timeout on
|
||||||
|
// every restart.
|
||||||
|
watchDone := make(chan struct{})
|
||||||
|
defer close(watchDone)
|
||||||
|
go func() {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
_ = stream.CloseResponse()
|
||||||
|
case <-watchDone:
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Sends are serialized through this channel: connect streams are not safe
|
||||||
|
// for concurrent writes, and heartbeats race with status reports.
|
||||||
|
out := make(chan *llmgwv1.AgentMessage, 32)
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
a.sendLoop(ctx, stream, out)
|
||||||
|
}()
|
||||||
|
defer wg.Wait()
|
||||||
|
|
||||||
|
for {
|
||||||
|
msg, err := stream.Receive()
|
||||||
|
if err != nil {
|
||||||
|
cancel()
|
||||||
|
switch {
|
||||||
|
case parent.Err() != nil:
|
||||||
|
// The agent is stopping; the stream ending is the consequence.
|
||||||
|
return nil
|
||||||
|
case errors.Is(err, io.EOF):
|
||||||
|
// The control plane closed the stream, which a restart does.
|
||||||
|
// Reconnect immediately rather than backing off.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// Everything else is worth both reporting and backing off from. A
|
||||||
|
// refused certificate arrives here, and an agent that cannot say
|
||||||
|
// why it is not registering is one an operator cannot fix.
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
a.handle(ctx, msg, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type sender interface {
|
||||||
|
Send(*llmgwv1.AgentMessage) error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Agent) sendLoop(ctx context.Context, s sender, out <-chan *llmgwv1.AgentMessage) {
|
||||||
|
t := time.NewTicker(a.cfg.HeartbeatInterval)
|
||||||
|
defer t.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case msg := <-out:
|
||||||
|
if err := s.Send(msg); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case <-t.C:
|
||||||
|
if err := s.Send(a.heartbeat()); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
|
||||||
|
a.mu.Lock()
|
||||||
|
inv := a.inv
|
||||||
|
a.mu.Unlock()
|
||||||
|
|
||||||
|
hb := &llmgwv1.Heartbeat{NodeId: a.cfg.NodeID}
|
||||||
|
for _, g := range inv.GPUs {
|
||||||
|
hb.GpuMemoryUsedMb = append(hb.GpuMemoryUsedMb, int32(g.MemoryUsedMB))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rescanned here rather than reused from registration. Disk is not a fact
|
||||||
|
// about a machine, it is a fact about a moment: a node that had 200 GB when
|
||||||
|
// its agent started can be full an hour later, and it is precisely the node
|
||||||
|
// busy filling up whose figure matters.
|
||||||
|
for _, st := range discovery.ScanStorage(a.cfg.ModelDirs) {
|
||||||
|
hb.Storage = append(hb.Storage, &llmgwv1.Storage{
|
||||||
|
Path: st.Path, FreeBytes: st.FreeBytes,
|
||||||
|
TotalBytes: st.TotalBytes, Shared: st.Shared,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// The file list is longer and changes rarely, so it goes only when it has.
|
||||||
|
// The flag is what makes an empty list mean "this node now holds nothing"
|
||||||
|
// rather than "nothing to report" — without it, the first heartbeat after
|
||||||
|
// a model was deleted would be indistinguishable from silence.
|
||||||
|
weights := discovery.ScanWeights(a.cfg.ModelDirs)
|
||||||
|
if fp := weightsFingerprint(weights); fp != a.lastWeights {
|
||||||
|
a.lastWeights = fp
|
||||||
|
hb.WeightsChanged = true
|
||||||
|
for _, w := range weights {
|
||||||
|
hb.Weights = append(hb.Weights, &llmgwv1.LocalWeights{
|
||||||
|
Path: w.Path, SizeBytes: w.SizeBytes, Shared: w.Shared,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Heartbeat{Heartbeat: hb}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// weightsFingerprint is enough to notice a model appearing, disappearing or
|
||||||
|
// finishing its download.
|
||||||
|
//
|
||||||
|
// Paths and sizes, because a fetch in progress has the same name as the file
|
||||||
|
// it will become — under a different name until it is renamed, but a shared
|
||||||
|
// filesystem can also grow a file in place.
|
||||||
|
func weightsFingerprint(ws []discovery.LocalWeights) string {
|
||||||
|
h := sha256.New()
|
||||||
|
for _, w := range ws {
|
||||||
|
fmt.Fprintf(h, "%s:%d\n", w.Path, w.SizeBytes)
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(h.Sum(nil))
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeWeights deletes cached model files the control plane says are no
|
||||||
|
// longer needed.
|
||||||
|
//
|
||||||
|
// Every path is checked here as well as there, and that duplication is the
|
||||||
|
// point. The control plane computes these from the catalog, but it is the node
|
||||||
|
// that holds the files, and a node must not be talkable into deleting a model
|
||||||
|
// an operator spent a week copying onto it — not by a control plane that is
|
||||||
|
// wrong, not by one that is newer than this agent, not by one that has been
|
||||||
|
// tampered with. So this deletes only what it can prove llmgw itself wrote:
|
||||||
|
// inside its own cache directory, under a name this system generates.
|
||||||
|
//
|
||||||
|
// Failures are logged and skipped rather than reported upward. A file that is
|
||||||
|
// already gone is the expected outcome of a retry, and one that will not
|
||||||
|
// delete is a disk to look at, not a reason to fail a command whose purpose
|
||||||
|
// was to free space.
|
||||||
|
func (a *Agent) removeWeights(paths []string) {
|
||||||
|
cache := a.cfg.WeightsCache
|
||||||
|
if cache == "" && len(a.cfg.ModelDirs) > 0 {
|
||||||
|
cache = a.cfg.ModelDirs[0]
|
||||||
|
}
|
||||||
|
if cache == "" {
|
||||||
|
a.log.Warn("asked to remove weights with no cache directory configured")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cacheAbs, err := filepath.Abs(cache)
|
||||||
|
if err != nil {
|
||||||
|
a.log.Warn("cannot resolve the cache directory", "dir", cache, "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, p := range paths {
|
||||||
|
abs, err := filepath.Abs(p)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Directly inside the cache directory, not merely under it: a nested
|
||||||
|
// path would let ".." or a symlinked subdirectory reach elsewhere, and
|
||||||
|
// nothing this package writes is nested anyway.
|
||||||
|
if filepath.Dir(abs) != cacheAbs {
|
||||||
|
a.log.Warn("refused to remove a file outside the weights cache", "path", p)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !weights.Owned(abs) {
|
||||||
|
a.log.Warn("refused to remove a file llmgw did not fetch", "path", p)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := os.Remove(abs); err != nil && !os.IsNotExist(err) {
|
||||||
|
a.log.Warn("could not remove cached weights", "path", p, "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
a.log.Info("removed cached weights no model asks for", "path", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out chan<- *llmgwv1.AgentMessage) {
|
||||||
|
switch m := msg.Msg.(type) {
|
||||||
|
case *llmgwv1.ControlMessage_Start:
|
||||||
|
go a.startShard(ctx, m.Start, out)
|
||||||
|
case *llmgwv1.ControlMessage_Stop:
|
||||||
|
go a.stopShard(ctx, m.Stop.GetShardId(), out)
|
||||||
|
case *llmgwv1.ControlMessage_Drain:
|
||||||
|
if err := a.sup.Drain(m.Drain.GetShardId()); err != nil {
|
||||||
|
a.log.Warn("draining failed", "shard", m.Drain.GetShardId(), "err", err)
|
||||||
|
}
|
||||||
|
case *llmgwv1.ControlMessage_RemoveWeights:
|
||||||
|
a.removeWeights(m.RemoveWeights.GetPaths())
|
||||||
|
case *llmgwv1.ControlMessage_Ack:
|
||||||
|
if !m.Ack.GetOk() {
|
||||||
|
a.log.Warn("control plane rejected a message",
|
||||||
|
"id", m.Ack.GetMessageId(), "err", m.Ack.GetError())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// startShard launches a shard and reports its progress.
|
||||||
|
//
|
||||||
|
// It runs in its own goroutine because model loading takes minutes, and the
|
||||||
|
// stream has to keep carrying heartbeats and other commands meanwhile.
|
||||||
|
func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<- *llmgwv1.AgentMessage) {
|
||||||
|
shardID, replicaID := m.GetShardId(), m.GetReplicaId()
|
||||||
|
|
||||||
|
a.mu.Lock()
|
||||||
|
a.shardReplica[shardID] = replicaID
|
||||||
|
a.mu.Unlock()
|
||||||
|
|
||||||
|
report := func(st supervisor.State, errMsg string, tail []string) {
|
||||||
|
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||||
|
NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID,
|
||||||
|
State: agentsvc.StateToWire(st), Error: errMsg, LastOutput: tail,
|
||||||
|
}}}
|
||||||
|
select {
|
||||||
|
case out <- msg:
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
spec, timeout := agentsvc.ShardSpecFromStart(m)
|
||||||
|
|
||||||
|
eng, err := a.reg.Get(spec.Profile.Engine)
|
||||||
|
if err != nil {
|
||||||
|
report(supervisor.Failed, err.Error(), nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
a.mu.Lock()
|
||||||
|
inv := a.inv
|
||||||
|
a.mu.Unlock()
|
||||||
|
|
||||||
|
// Validate before launching. An engine that refuses a profile does so after
|
||||||
|
// minutes of loading; catching it here costs nothing.
|
||||||
|
if err := eng.Validate(spec.Profile, gpuInfo(inv, spec.GPUIndices)); err != nil {
|
||||||
|
report(supervisor.Failed, err.Error(), nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// The weights, before anything is launched. A source that is already a
|
||||||
|
// path is checked and used; one that names a remote file is fetched once
|
||||||
|
// and shared by every shard that wants it.
|
||||||
|
//
|
||||||
|
// Checked here rather than left to the engine because the engine's answer
|
||||||
|
// arrives minutes into a load, in its own words, on a node nobody is
|
||||||
|
// watching — and because eight shards starting together would otherwise
|
||||||
|
// each fetch the same file.
|
||||||
|
if path, err := a.weights().Ensure(ctx, spec.Profile.ModelPath); err != nil {
|
||||||
|
report(supervisor.Failed, err.Error(), nil)
|
||||||
|
return
|
||||||
|
} else {
|
||||||
|
spec.Profile.ModelPath = path
|
||||||
|
}
|
||||||
|
|
||||||
|
cmds, err := eng.Commands(spec)
|
||||||
|
if err != nil {
|
||||||
|
report(supervisor.Failed, err.Error(), nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
sh, err := a.sup.Start(ctx, shardID, spec, cmds)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, supervisor.ErrAlreadyRunning) {
|
||||||
|
// Commands are idempotent: the control plane retries, and a repeat
|
||||||
|
// must not produce a second process (plan §5.6).
|
||||||
|
report(sh.Status().State, "", nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
report(supervisor.Failed, err.Error(), sh.Logs(10))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
report(supervisor.Starting, "", nil)
|
||||||
|
|
||||||
|
// Only a rank that serves HTTP can be health-checked. A worker is judged by
|
||||||
|
// staying alive, and the replica's real readiness is the leader answering,
|
||||||
|
// which only happens once every worker has joined (plan §11, phase 4).
|
||||||
|
if eng.ServesHTTP(spec) {
|
||||||
|
base := fmt.Sprintf("http://127.0.0.1:%d", spec.HTTPPort)
|
||||||
|
err = a.sup.WaitReady(ctx, sh, base, spec.Profile.ServedModelName, timeout)
|
||||||
|
} else {
|
||||||
|
err = a.sup.WaitAlive(ctx, sh)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
report(supervisor.Failed, err.Error(), sh.Logs(20))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
report(supervisor.Ready, "", nil)
|
||||||
|
|
||||||
|
// Watch for the engine dying later. Nothing else would notice: the router
|
||||||
|
// would simply see failing requests.
|
||||||
|
go a.watch(ctx, sh, replicaID, shardID, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// watch reports a shard that exits after it was ready.
|
||||||
|
func (a *Agent) watch(ctx context.Context, sh *supervisor.Shard, replicaID, shardID string, out chan<- *llmgwv1.AgentMessage) {
|
||||||
|
t := time.NewTicker(time.Second)
|
||||||
|
defer t.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-t.C:
|
||||||
|
st := sh.Status()
|
||||||
|
if !st.State.Terminal() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||||
|
NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID,
|
||||||
|
State: agentsvc.StateToWire(st.State), Error: st.Err,
|
||||||
|
ExitCode: int32(st.ExitCode), LastOutput: sh.Logs(20),
|
||||||
|
}}}
|
||||||
|
select {
|
||||||
|
case out <- msg:
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Agent) stopShard(ctx context.Context, shardID string, out chan<- *llmgwv1.AgentMessage) {
|
||||||
|
a.mu.Lock()
|
||||||
|
replicaID := a.shardReplica[shardID]
|
||||||
|
delete(a.shardReplica, shardID)
|
||||||
|
a.mu.Unlock()
|
||||||
|
|
||||||
|
if err := a.sup.Stop(ctx, shardID); err != nil {
|
||||||
|
a.log.Warn("stopping a shard failed", "shard", shardID, "err", err)
|
||||||
|
}
|
||||||
|
a.sup.Remove(shardID)
|
||||||
|
|
||||||
|
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||||
|
NodeId: a.cfg.NodeID, ReplicaId: replicaID, ShardId: shardID,
|
||||||
|
State: agentsvc.StateToWire(supervisor.Stopped),
|
||||||
|
}}}
|
||||||
|
select {
|
||||||
|
case out <- msg:
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// gpuInfo narrows the inventory to the GPUs a shard will use.
|
||||||
|
func gpuInfo(inv discovery.Inventory, indices []int) []engines.GPUInfo {
|
||||||
|
want := map[int]struct{}{}
|
||||||
|
for _, i := range indices {
|
||||||
|
want[i] = struct{}{}
|
||||||
|
}
|
||||||
|
var out []engines.GPUInfo
|
||||||
|
for _, g := range inv.GPUs {
|
||||||
|
if _, ok := want[g.Index]; !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, engines.GPUInfo{
|
||||||
|
Index: g.Index, Name: g.Name,
|
||||||
|
ComputeCapability: g.ComputeCapability, MemoryTotalMB: g.MemoryTotalMB,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Client returns the client the agent reaches the control plane with.
|
||||||
|
//
|
||||||
|
// Both branches speak HTTP/2, which connect's bidirectional streams require and
|
||||||
|
// which Go will not do by default in either case: without TLS it has to be
|
||||||
|
// asked explicitly, and with TLS it has to be negotiated through ALPN.
|
||||||
|
//
|
||||||
|
// The plaintext branch exists for local runs against sim-engines. Anything
|
||||||
|
// using it must stay on a trusted network, because nothing there stops a node
|
||||||
|
// from claiming to be another one.
|
||||||
|
func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) {
|
||||||
|
if !cfg.Enabled() {
|
||||||
|
return &http.Client{
|
||||||
|
Transport: &http2.Transport{
|
||||||
|
AllowHTTP: true,
|
||||||
|
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
|
||||||
|
return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServerName is left to the transport, which takes it from the address
|
||||||
|
// being dialled. Hardcoding it here would break the moment the control
|
||||||
|
// plane moved.
|
||||||
|
tc, err := cfg.ClientConfig("")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &http.Client{Transport: &http2.Transport{TLSClientConfig: tc}}, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,738 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/tls"
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/net/http2"
|
||||||
|
"golang.org/x/net/http2/h2c"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||||
|
)
|
||||||
|
|
||||||
|
// statusRecorder captures what the control plane hears about shards, and what
|
||||||
|
// it is asked to adopt.
|
||||||
|
type statusRecorder struct {
|
||||||
|
ch chan status
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
calls []adoptCall
|
||||||
|
adopted []reconciler.AdoptedShard
|
||||||
|
unclaim []string // shard ids to report back as unclaimed
|
||||||
|
}
|
||||||
|
|
||||||
|
type status struct {
|
||||||
|
replicaID, shardID string
|
||||||
|
state supervisor.State
|
||||||
|
err string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRecorder() *statusRecorder { return &statusRecorder{ch: make(chan status, 128)} }
|
||||||
|
|
||||||
|
func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
|
||||||
|
select {
|
||||||
|
case r.ch <- status{replicaID, shardID, st, errMsg}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *statusRecorder) Adopt(nodeID string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
r.calls = append(r.calls, adoptCall{nodeID: nodeID, shards: shards})
|
||||||
|
r.adopted = append(r.adopted, shards...)
|
||||||
|
|
||||||
|
var unclaimed []reconciler.AdoptedShard
|
||||||
|
for _, s := range shards {
|
||||||
|
if slices.Contains(r.unclaim, s.ShardID) {
|
||||||
|
unclaimed = append(unclaimed, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return unclaimed
|
||||||
|
}
|
||||||
|
|
||||||
|
// adoptCall records that the service reached the reconciler at all, which an
|
||||||
|
// empty shard list used to prevent.
|
||||||
|
type adoptCall struct {
|
||||||
|
nodeID string
|
||||||
|
shards []reconciler.AdoptedShard
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *statusRecorder) adoptCalls() []adoptCall {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
return slices.Clone(r.calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *statusRecorder) adoptedShards() []reconciler.AdoptedShard {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
return slices.Clone(r.adopted)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *statusRecorder) markUnclaimed(ids ...string) {
|
||||||
|
r.mu.Lock()
|
||||||
|
r.unclaim = append(r.unclaim, ids...)
|
||||||
|
r.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *statusRecorder) await(t *testing.T, limit time.Duration, pred func(status) bool) status {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.After(limit)
|
||||||
|
var seen []status
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case s := <-r.ch:
|
||||||
|
seen = append(seen, s)
|
||||||
|
if pred(s) {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
case <-deadline:
|
||||||
|
t.Fatalf("no matching status within %v; saw %+v", limit, seen)
|
||||||
|
return status{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeEngineScript writes a script that stands in for vLLM.
|
||||||
|
//
|
||||||
|
// It parses the real --port flag the vLLM adapter generates rather than being
|
||||||
|
// told out of band, so the adapter's argument construction is exercised too.
|
||||||
|
func fakeEngineScript(t *testing.T, body string) string {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(t.TempDir(), "fake-vllm")
|
||||||
|
script := `#!/bin/sh
|
||||||
|
port=""
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--port) port="$2"; shift 2 ;;
|
||||||
|
*) shift ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
` + body + "\n"
|
||||||
|
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
// servingEngine answers /health and /v1/models, which is what readiness needs.
|
||||||
|
func servingEngine(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
py, err := exec.LookPath("python3")
|
||||||
|
if err != nil {
|
||||||
|
t.Skip("python3 is required for the fake engine")
|
||||||
|
}
|
||||||
|
return fakeEngineScript(t, `exec `+py+` -c '
|
||||||
|
import http.server, json, sys
|
||||||
|
port = int(sys.argv[1])
|
||||||
|
class H(http.server.BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self):
|
||||||
|
if self.path == "/health":
|
||||||
|
self.send_response(200); self.end_headers(); return
|
||||||
|
if self.path == "/v1/models":
|
||||||
|
b = json.dumps({"object":"list","data":[{"id":"test-model"}]}).encode()
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("content-type","application/json")
|
||||||
|
self.send_header("content-length", str(len(b)))
|
||||||
|
self.end_headers(); self.wfile.write(b); return
|
||||||
|
self.send_response(404); self.end_headers()
|
||||||
|
def log_message(self, *a): pass
|
||||||
|
http.server.HTTPServer(("127.0.0.1", port), H).serve_forever()
|
||||||
|
' "$port"`)
|
||||||
|
}
|
||||||
|
|
||||||
|
type harness struct {
|
||||||
|
svc *agentsvc.Service
|
||||||
|
inv *inventory.Store
|
||||||
|
rec *statusRecorder
|
||||||
|
srv *httptest.Server
|
||||||
|
ag *Agent
|
||||||
|
ln *dropListener
|
||||||
|
// streams counts how many times an agent has opened the Connect stream,
|
||||||
|
// which is the only honest way to tell a reconnect from a stale entry.
|
||||||
|
streams *atomic.Int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// dropListener tracks accepted connections so a test can sever them.
|
||||||
|
//
|
||||||
|
// httptest's CloseClientConnections does not work here: h2c with prior
|
||||||
|
// knowledge hijacks the connection, and httptest stops tracking hijacked
|
||||||
|
// connections. A test relying on it would pass without ever reconnecting.
|
||||||
|
type dropListener struct {
|
||||||
|
net.Listener
|
||||||
|
mu sync.Mutex
|
||||||
|
conns []net.Conn
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *dropListener) Accept() (net.Conn, error) {
|
||||||
|
c, err := l.Listener.Accept()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
l.mu.Lock()
|
||||||
|
l.conns = append(l.conns, c)
|
||||||
|
l.mu.Unlock()
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// drop severs every connection, as a control plane restart would.
|
||||||
|
func (l *dropListener) drop() {
|
||||||
|
l.mu.Lock()
|
||||||
|
conns := l.conns
|
||||||
|
l.conns = nil
|
||||||
|
l.mu.Unlock()
|
||||||
|
for _, c := range conns {
|
||||||
|
_ = c.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// start wires a real agent to a real service over h2c, so this exercises the
|
||||||
|
// protocol rather than two structs calling each other.
|
||||||
|
func start(t *testing.T, cfgFns ...func(*Config)) *harness {
|
||||||
|
t.Helper()
|
||||||
|
return startWith(t, slog.New(slog.DiscardHandler), cfgFns...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// startLogged is start with logging on, for diagnosing protocol behaviour.
|
||||||
|
func startLogged(t *testing.T, cfgFns ...func(*Config)) *harness {
|
||||||
|
t.Helper()
|
||||||
|
return startWith(t, slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})), cfgFns...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func startWith(t *testing.T, log *slog.Logger, cfgFns ...func(*Config)) *harness {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
inv := inventory.NewStore()
|
||||||
|
rec := newRecorder()
|
||||||
|
svc := agentsvc.New(inv, rec, false, log)
|
||||||
|
|
||||||
|
path, handler := svc.Handler()
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
mux.Handle(path, handler)
|
||||||
|
|
||||||
|
var streams atomic.Int64
|
||||||
|
counting := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
streams.Add(1)
|
||||||
|
mux.ServeHTTP(w, r)
|
||||||
|
})
|
||||||
|
|
||||||
|
raw, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ln := &dropListener{Listener: raw}
|
||||||
|
// connect's bidirectional streams need HTTP/2; h2c provides it without TLS.
|
||||||
|
srv := httptest.NewUnstartedServer(h2c.NewHandler(counting, &http2.Server{}))
|
||||||
|
srv.Listener = ln
|
||||||
|
srv.Start()
|
||||||
|
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.NodeID = "n1"
|
||||||
|
cfg.Server = srv.URL
|
||||||
|
cfg.MgmtAddr = "127.0.0.1"
|
||||||
|
cfg.HeartbeatInterval = 50 * time.Millisecond
|
||||||
|
cfg.ReconnectBase = 20 * time.Millisecond
|
||||||
|
cfg.ReconnectMax = 100 * time.Millisecond
|
||||||
|
cfg.Discovery.SkipGPUs = true
|
||||||
|
cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent")
|
||||||
|
cfg.Superv = supervisor.Config{
|
||||||
|
WorkDir: t.TempDir(), LogLines: 64,
|
||||||
|
StopTimeout: 2 * time.Second, HealthInterval: 20 * time.Millisecond,
|
||||||
|
}
|
||||||
|
for _, f := range cfgFns {
|
||||||
|
f(&cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
client := &http.Client{Transport: &http2.Transport{
|
||||||
|
AllowHTTP: true,
|
||||||
|
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
|
||||||
|
return (&net.Dialer{}).DialContext(ctx, network, addr)
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
|
||||||
|
ag, err := New(cfg, client, log)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
go func() { _ = ag.Run(ctx) }()
|
||||||
|
|
||||||
|
h := &harness{svc: svc, inv: inv, rec: rec, srv: srv, ag: ag, ln: ln, streams: &streams}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
cancel()
|
||||||
|
ag.Supervisor().StopAll(context.Background())
|
||||||
|
srv.Close()
|
||||||
|
})
|
||||||
|
h.waitRegistered(t)
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *harness) waitRegistered(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(10 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if _, ok := h.inv.Get("n1"); ok && len(h.svc.ConnectedNodes()) == 1 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatal("the agent never registered")
|
||||||
|
}
|
||||||
|
|
||||||
|
// shardCommand builds a single-node vLLM shard on a free port.
|
||||||
|
//
|
||||||
|
// The model path is a real file. It used to be "/models/x", which stopped
|
||||||
|
// working once the agent began checking that weights exist before launching an
|
||||||
|
// engine — the check doing exactly what it is for, on a test that was relying
|
||||||
|
// on nobody looking.
|
||||||
|
func shardCommand(t *testing.T, shardID string) reconciler.ShardCommand {
|
||||||
|
t.Helper()
|
||||||
|
modelPath := filepath.Join(t.TempDir(), "model.gguf")
|
||||||
|
if err := os.WriteFile(modelPath, []byte("weights"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return reconciler.ShardCommand{
|
||||||
|
ReplicaID: "rep-1", ShardID: shardID, NodeID: "n1",
|
||||||
|
Spec: engines.ShardSpec{
|
||||||
|
ReplicaID: "rep-1", ShardID: shardID, Rank: 0, NNodes: 1,
|
||||||
|
GPUIndices: []int{0}, HTTPPort: freePort(t),
|
||||||
|
Profile: engines.Profile{
|
||||||
|
Name: "p", Engine: "vllm", TP: 1,
|
||||||
|
ModelPath: modelPath, ServedModelName: "test-model",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
StartupTimeout: 15 * time.Second,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func freePort(t *testing.T) int {
|
||||||
|
t.Helper()
|
||||||
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer ln.Close()
|
||||||
|
return ln.Addr().(*net.TCPAddr).Port
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRegistrationCarriesInventory(t *testing.T) {
|
||||||
|
// The node must actually have the adapters it declares. It used to declare
|
||||||
|
// mlx5_0 and mlx5_1 on a node whose sysfs held neither, which passed
|
||||||
|
// because nothing compared the two — and that is the gap a real cluster
|
||||||
|
// would have found as a replica hanging on its first all-reduce.
|
||||||
|
ibRoot := ibTreeWithActivePorts(t, "mlx5_0", "mlx5_1")
|
||||||
|
h := start(t, func(c *Config) {
|
||||||
|
c.Hostname = "gpu-a-01"
|
||||||
|
c.Labels = map[string]string{"rack": "r12"}
|
||||||
|
c.Discovery.IBRoot = ibRoot
|
||||||
|
c.Fabrics = []inventory.FabricAttachment{
|
||||||
|
{Fabric: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}, RateGbps: 200},
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
n, ok := h.inv.Get("n1")
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("node missing from inventory")
|
||||||
|
}
|
||||||
|
if n.Hostname != "gpu-a-01" || n.MgmtAddr != "127.0.0.1" {
|
||||||
|
t.Errorf("identity wrong: %+v", n)
|
||||||
|
}
|
||||||
|
if n.Labels["rack"] != "r12" {
|
||||||
|
t.Errorf("labels lost: %v", n.Labels)
|
||||||
|
}
|
||||||
|
// The fabric label cannot be inferred from the wire, so losing it in
|
||||||
|
// transit would silently disable multi-node placement.
|
||||||
|
if !n.HasFabric("ib-a") {
|
||||||
|
t.Fatalf("fabric attachment lost: %+v", n.Fabrics)
|
||||||
|
}
|
||||||
|
if got := n.HCAsFor("ib-a"); !slices.Equal(got, []string{"mlx5_0", "mlx5_1"}) {
|
||||||
|
t.Errorf("HCAs=%v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHeartbeatKeepsNodeAlive(t *testing.T) {
|
||||||
|
h := start(t)
|
||||||
|
|
||||||
|
n, _ := h.inv.Get("n1")
|
||||||
|
first := n.LastSeen
|
||||||
|
|
||||||
|
time.Sleep(200 * time.Millisecond)
|
||||||
|
n, _ = h.inv.Get("n1")
|
||||||
|
if !n.LastSeen.After(first) {
|
||||||
|
t.Error("heartbeats are not refreshing LastSeen; the node would be declared lost")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStartAndStopShardEndToEnd(t *testing.T) {
|
||||||
|
h := start(t, func(c *Config) {
|
||||||
|
c.Engines.VLLM.Bin = servingEngine(t)
|
||||||
|
})
|
||||||
|
|
||||||
|
cmd := shardCommand(t, "shard-a")
|
||||||
|
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := h.rec.await(t, 30*time.Second, func(s status) bool {
|
||||||
|
return s.shardID == "shard-a" && s.state == supervisor.Ready
|
||||||
|
})
|
||||||
|
if got.replicaID != "rep-1" {
|
||||||
|
t.Errorf("replica id lost in transit: %q", got.replicaID)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := h.svc.StopShard(t.Context(), "n1", "shard-a"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
h.rec.await(t, 15*time.Second, func(s status) bool {
|
||||||
|
return s.shardID == "shard-a" && s.state == supervisor.Stopped
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFailedShardCarriesDiagnosis(t *testing.T) {
|
||||||
|
// When a model fails to load the engine's output is the only diagnosis, so
|
||||||
|
// it has to survive the trip to the control plane.
|
||||||
|
h := start(t, func(c *Config) {
|
||||||
|
c.Engines.VLLM.Bin = fakeEngineScript(t, `echo 'CUDA out of memory' >&2; exit 1`)
|
||||||
|
})
|
||||||
|
|
||||||
|
if err := h.svc.StartShard(t.Context(), shardCommand(t, "shard-bad")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := h.rec.await(t, 30*time.Second, func(s status) bool {
|
||||||
|
return s.shardID == "shard-bad" && s.state == supervisor.Failed
|
||||||
|
})
|
||||||
|
if got.err == "" {
|
||||||
|
t.Error("the failure carried no error message")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEngineThatNeverBecomesReady(t *testing.T) {
|
||||||
|
// A process that runs but never serves must time out rather than hang the
|
||||||
|
// replica forever.
|
||||||
|
h := start(t, func(c *Config) {
|
||||||
|
c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 60`)
|
||||||
|
})
|
||||||
|
|
||||||
|
cmd := shardCommand(t, "shard-stuck")
|
||||||
|
cmd.StartupTimeout = 500 * time.Millisecond
|
||||||
|
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
h.rec.await(t, 20*time.Second, func(s status) bool {
|
||||||
|
return s.shardID == "shard-stuck" && s.state == supervisor.Failed
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidationRejectsBeforeLaunch(t *testing.T) {
|
||||||
|
// An engine that refuses a profile does so after minutes of loading;
|
||||||
|
// catching it at dispatch costs nothing.
|
||||||
|
h := start(t, func(c *Config) {
|
||||||
|
c.Engines.VLLM.Bin = servingEngine(t)
|
||||||
|
})
|
||||||
|
|
||||||
|
cmd := shardCommand(t, "shard-invalid")
|
||||||
|
cmd.Spec.Profile.ModelPath = "" // validation requires one
|
||||||
|
|
||||||
|
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := h.rec.await(t, 15*time.Second, func(s status) bool {
|
||||||
|
return s.shardID == "shard-invalid" && s.state == supervisor.Failed
|
||||||
|
})
|
||||||
|
if got.err == "" {
|
||||||
|
t.Error("validation failure carried no reason")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDispatchToUnconnectedNode(t *testing.T) {
|
||||||
|
h := start(t)
|
||||||
|
|
||||||
|
err := h.svc.StartShard(t.Context(), reconciler.ShardCommand{NodeID: "nowhere", ShardID: "x"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("dispatching to an unconnected node succeeded")
|
||||||
|
}
|
||||||
|
if !errors.Is(err, agentsvc.ErrNotConnected) {
|
||||||
|
t.Errorf("err=%v, expected ErrNotConnected", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAgentReconnects(t *testing.T) {
|
||||||
|
// A control plane restart must not require touching every GPU node.
|
||||||
|
h := start(t)
|
||||||
|
|
||||||
|
before := h.streams.Load()
|
||||||
|
h.ln.drop()
|
||||||
|
|
||||||
|
// Counting streams, not connected nodes: a stale entry in the connection
|
||||||
|
// map would make a node look connected without any reconnect having
|
||||||
|
// happened, and an assertion on it would pass vacuously.
|
||||||
|
deadline := time.Now().Add(15 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if h.streams.Load() > before && len(h.svc.ConnectedNodes()) == 1 {
|
||||||
|
if err := h.svc.DrainShard(t.Context(), "n1", "nothing"); err != nil {
|
||||||
|
t.Fatalf("the reconnected stream does not accept commands: %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatalf("the agent never reconnected (streams %d -> %d)", before, h.streams.Load())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIdempotentStart(t *testing.T) {
|
||||||
|
// The control plane retries commands; a repeat must not start a second
|
||||||
|
// process (plan §5.6).
|
||||||
|
h := start(t, func(c *Config) {
|
||||||
|
c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 30`)
|
||||||
|
})
|
||||||
|
|
||||||
|
cmd := shardCommand(t, "shard-idem")
|
||||||
|
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
h.rec.await(t, 15*time.Second, func(s status) bool {
|
||||||
|
return s.shardID == "shard-idem" && s.state == supervisor.Starting
|
||||||
|
})
|
||||||
|
|
||||||
|
sup := h.ag.Supervisor()
|
||||||
|
sh, err := sup.Get("shard-idem")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pid := sh.Status().PID
|
||||||
|
|
||||||
|
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(400 * time.Millisecond)
|
||||||
|
|
||||||
|
sh2, err := sup.Get("shard-idem")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if sh2.Status().PID != pid {
|
||||||
|
t.Errorf("a second process was started: pid %d then %d", pid, sh2.Status().PID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunningShardsReportedOnReconnect covers the control plane restart hole
|
||||||
|
// (plan §6.5): a reconnecting agent has to describe what it is already running,
|
||||||
|
// or the control plane concludes the cluster is empty and places duplicates
|
||||||
|
// while the originals keep holding GPUs.
|
||||||
|
func TestRunningShardsReportedOnReconnect(t *testing.T) {
|
||||||
|
h := start(t, func(c *Config) {
|
||||||
|
c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 60`)
|
||||||
|
})
|
||||||
|
|
||||||
|
cmd := shardCommand(t, "shard-live")
|
||||||
|
cmd.Pool = "ib-a/h100-80g"
|
||||||
|
cmd.Fabric = "ib-a"
|
||||||
|
cmd.Spec.HCAs = []string{"mlx5_0"}
|
||||||
|
|
||||||
|
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
h.rec.await(t, 15*time.Second, func(s status) bool {
|
||||||
|
return s.shardID == "shard-live" && s.state == supervisor.Starting
|
||||||
|
})
|
||||||
|
|
||||||
|
// Drop the stream, as a control plane restart would.
|
||||||
|
h.ln.drop()
|
||||||
|
|
||||||
|
deadline := time.Now().Add(15 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if got := h.rec.adoptedShards(); len(got) > 0 {
|
||||||
|
a := got[0]
|
||||||
|
if a.ShardID != "shard-live" || a.ReplicaID != "rep-1" {
|
||||||
|
t.Fatalf("wrong shard reported: %+v", a)
|
||||||
|
}
|
||||||
|
// Everything the control plane needs to rebuild the replica.
|
||||||
|
if a.Model != "test-model" || a.Profile != "p" {
|
||||||
|
t.Errorf("catalog identity lost: model=%q profile=%q", a.Model, a.Profile)
|
||||||
|
}
|
||||||
|
if a.Pool != "ib-a/h100-80g" || a.Fabric != "ib-a" {
|
||||||
|
t.Errorf("pool/fabric lost: %q %q", a.Pool, a.Fabric)
|
||||||
|
}
|
||||||
|
if !slices.Equal(a.HCAs, []string{"mlx5_0"}) {
|
||||||
|
t.Errorf("HCAs lost: %v", a.HCAs)
|
||||||
|
}
|
||||||
|
if a.Endpoint == "" {
|
||||||
|
t.Error("rank 0 reported no endpoint; the router would have nowhere to send")
|
||||||
|
}
|
||||||
|
if len(a.GPUIndices) == 0 {
|
||||||
|
t.Error("GPU indices lost; the inventory could not reclaim them")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatal("the reconnecting agent never reported its running shard")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUnclaimedShardIsStopped covers the other half: a shard whose model left
|
||||||
|
// the catalog is holding GPUs nothing will ever route to.
|
||||||
|
func TestUnclaimedShardIsStopped(t *testing.T) {
|
||||||
|
h := start(t, func(c *Config) {
|
||||||
|
c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 60`)
|
||||||
|
})
|
||||||
|
|
||||||
|
cmd := shardCommand(t, "shard-orphaned")
|
||||||
|
if err := h.svc.StartShard(t.Context(), cmd); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
h.rec.await(t, 15*time.Second, func(s status) bool {
|
||||||
|
return s.shardID == "shard-orphaned" && s.state == supervisor.Starting
|
||||||
|
})
|
||||||
|
|
||||||
|
sh, err := h.ag.Supervisor().Get("shard-orphaned")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pid := sh.Status().PID
|
||||||
|
|
||||||
|
// The control plane will refuse to claim it on reconnect.
|
||||||
|
h.rec.markUnclaimed("shard-orphaned")
|
||||||
|
h.ln.drop()
|
||||||
|
|
||||||
|
deadline := time.Now().Add(20 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if !processAliveForTest(pid) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Error("an unclaimed shard kept running and holding its GPUs")
|
||||||
|
}
|
||||||
|
|
||||||
|
// processAliveForTest mirrors the supervisor's liveness check.
|
||||||
|
func processAliveForTest(pid int) bool {
|
||||||
|
return syscall.Kill(pid, 0) == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAShardWithMissingWeightsFailsBeforeLaunching(t *testing.T) {
|
||||||
|
// The commonest setup mistake on a fresh node. Left to the engine it
|
||||||
|
// surfaces minutes into a load, in the engine's own words, on a machine
|
||||||
|
// nobody is watching — so it is caught here, where the message can say
|
||||||
|
// what is actually wrong.
|
||||||
|
a := &Agent{cfg: Config{WeightsCache: t.TempDir()}, log: slog.New(slog.DiscardHandler)}
|
||||||
|
_, err := a.weights().Ensure(t.Context(), filepath.Join(t.TempDir(), "not-there.gguf"))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a missing weights file was accepted")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "not-there.gguf") {
|
||||||
|
t.Errorf("the error does not name the file: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheCacheDefaultsToTheFirstModelDirectory(t *testing.T) {
|
||||||
|
// A node that has said where its models live has already answered this;
|
||||||
|
// asking twice is a second thing to get out of step.
|
||||||
|
dir := t.TempDir()
|
||||||
|
a := &Agent{cfg: Config{ModelDirs: []string{dir, "/elsewhere"}}, log: slog.New(slog.DiscardHandler)}
|
||||||
|
if got := a.weights().CacheDir; got != dir {
|
||||||
|
t.Errorf("cache = %q, want the first model directory %q", got, dir)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ibTreeWithActivePorts builds a sysfs InfiniBand tree with the named adapters,
|
||||||
|
// each with one active port, shaped like the real thing.
|
||||||
|
func ibTreeWithActivePorts(t *testing.T, names ...string) string {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
for _, name := range names {
|
||||||
|
dir := filepath.Join(root, name, "ports", "1")
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "gids"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for file, content := range map[string]string{
|
||||||
|
"state": "4: ACTIVE",
|
||||||
|
"phys_state": "5: LinkUp",
|
||||||
|
"rate": "200 Gb/sec (4X HDR)",
|
||||||
|
"link_layer": "InfiniBand",
|
||||||
|
"lid": "0x1a",
|
||||||
|
"gids/0": "fec0:0000:0000:000a:0002:c903:00a0:1234",
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, file), []byte(content+"\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return root
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFabricWhoseAdaptersAreAbsentIsNotAdvertised(t *testing.T) {
|
||||||
|
// The failure this prevents: a typo, or a card that enumerated differently
|
||||||
|
// after a reboot, produced NCCL_IB_HCA naming an adapter that is not there.
|
||||||
|
// The replica starts, passes its health checks, and hangs on its first
|
||||||
|
// all-reduce — the exact case the fabric invariant exists to prevent,
|
||||||
|
// arriving through a door nobody was watching.
|
||||||
|
h := start(t, func(c *Config) {
|
||||||
|
c.Discovery.IBRoot = ibTreeWithActivePorts(t, "mlx5_0")
|
||||||
|
c.Fabrics = []inventory.FabricAttachment{
|
||||||
|
{Fabric: "ib-typo", HCAs: []string{"mlx5_9"}, RateGbps: 200},
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
n, ok := h.inv.Get("n1")
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("node missing from inventory")
|
||||||
|
}
|
||||||
|
if n.HasFabric("ib-typo") {
|
||||||
|
t.Errorf("a fabric with no usable adapter was advertised: %+v", n.Fabrics)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFabricRateIsTheSlowestActiveLink(t *testing.T) {
|
||||||
|
// A collective runs at the speed of its slowest link, so reporting the
|
||||||
|
// fastest would have the scheduler believe a fabric is quicker than it is.
|
||||||
|
root := t.TempDir()
|
||||||
|
for name, rate := range map[string]string{"mlx5_0": "200", "mlx5_1": "100"} {
|
||||||
|
dir := filepath.Join(root, name, "ports", "1")
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "gids"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for file, content := range map[string]string{
|
||||||
|
"state": "4: ACTIVE", "phys_state": "5: LinkUp",
|
||||||
|
"rate": rate + " Gb/sec (4X HDR)", "link_layer": "InfiniBand",
|
||||||
|
"lid": "0x1a", "gids/0": "fec0::1",
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, file), []byte(content+"\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
h := start(t, func(c *Config) {
|
||||||
|
c.Discovery.IBRoot = root
|
||||||
|
c.Fabrics = []inventory.FabricAttachment{
|
||||||
|
{Fabric: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}},
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
n, _ := h.inv.Get("n1")
|
||||||
|
for _, f := range n.Fabrics {
|
||||||
|
if f.Fabric == "ib-a" && f.RateGbps != 100 {
|
||||||
|
t.Errorf("fabric rate = %d, want the slower link's 100", f.RateGbps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrNoNVML means this binary was built without the NVML backend.
|
||||||
|
//
|
||||||
|
// It is declared here rather than in the build-tagged file so callers — and
|
||||||
|
// tests — can refer to it in either build.
|
||||||
|
var ErrNoNVML = errors.New("built without the nvml tag; rebuild with -tags nvml for NVLink topology")
|
||||||
|
|
||||||
|
// Options configure a full inventory scan.
|
||||||
|
type Options struct {
|
||||||
|
// NvidiaSMI is the path to nvidia-smi; empty means look it up on PATH.
|
||||||
|
// Worth configuring on nodes where the driver lives outside the default
|
||||||
|
// paths, which containerized driver installs often do.
|
||||||
|
NvidiaSMI string `koanf:"nvidia_smi"`
|
||||||
|
// IBRoot is the sysfs InfiniBand directory.
|
||||||
|
IBRoot string `koanf:"ib_root"`
|
||||||
|
// PreferNVML uses the NVML backend when this binary has one. It is the only
|
||||||
|
// source of NVLink topology.
|
||||||
|
PreferNVML bool `koanf:"prefer_nvml"`
|
||||||
|
// SkipGPUs omits GPU discovery, for nodes that have none.
|
||||||
|
SkipGPUs bool `koanf:"skip_gpus"`
|
||||||
|
// ModelDirs are scanned for weights. Empty means do not look.
|
||||||
|
ModelDirs []string `koanf:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Discover builds the node's inventory.
|
||||||
|
//
|
||||||
|
// When NVML is preferred but fails, it falls back to nvidia-smi and says so in
|
||||||
|
// Inventory.Source. The fallback is logged rather than silent: on a node that
|
||||||
|
// is supposed to have working NVML, a quiet downgrade hides a broken driver and
|
||||||
|
// costs the scheduler its NVLink topology.
|
||||||
|
func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, error) {
|
||||||
|
if log == nil {
|
||||||
|
log = slog.Default()
|
||||||
|
}
|
||||||
|
inv := Inventory{Source: "none"}
|
||||||
|
// First, because it has nothing to do with GPUs and every path below can
|
||||||
|
// return early. Read after them, a node with no nvidia-smi reported no
|
||||||
|
// memory at all: GPU discovery fails, Discover returns, and the fact that
|
||||||
|
// has nothing to do with cards goes with it. That is exactly the node whose
|
||||||
|
// memory matters most, since a CPU profile is what it will be asked to run.
|
||||||
|
inv.MemoryTotalMB = MemoryTotalMB()
|
||||||
|
// Also before the GPU paths, and for the same reason: which weights a node
|
||||||
|
// holds has nothing to do with its cards, and a node with none is exactly
|
||||||
|
// where a CPU profile will look for them.
|
||||||
|
inv.Weights = ScanWeights(opts.ModelDirs)
|
||||||
|
// And the disk they sit on. A node can hold a model today and be unable to
|
||||||
|
// take the next one, which the file list alone never shows.
|
||||||
|
inv.Storage = ScanStorage(opts.ModelDirs)
|
||||||
|
|
||||||
|
if !opts.SkipGPUs {
|
||||||
|
switch {
|
||||||
|
case opts.PreferNVML && nvmlAvailable:
|
||||||
|
gpus, err := DiscoverGPUsNVML(ctx)
|
||||||
|
if err == nil {
|
||||||
|
inv.GPUs, inv.Source = gpus, "nvml"
|
||||||
|
break
|
||||||
|
}
|
||||||
|
log.Warn("NVML discovery failed, falling back to nvidia-smi; "+
|
||||||
|
"NVLink topology will be unavailable", "err", err)
|
||||||
|
fallthrough
|
||||||
|
default:
|
||||||
|
gpus, err := DiscoverGPUsSMI(ctx, opts.NvidiaSMI)
|
||||||
|
if err != nil {
|
||||||
|
return inv, fmt.Errorf("gpu discovery: %w", err)
|
||||||
|
}
|
||||||
|
inv.GPUs, inv.Source = gpus, "nvidia-smi"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
hcas, err := DiscoverIB(opts.IBRoot)
|
||||||
|
if err != nil {
|
||||||
|
return inv, fmt.Errorf("infiniband discovery: %w", err)
|
||||||
|
}
|
||||||
|
inv.HCAs = hcas
|
||||||
|
return inv, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NVMLSupported reports whether this binary includes the NVML backend.
|
||||||
|
func NVMLSupported() bool { return nvmlAvailable }
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func fakeSMI(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
stub := filepath.Join(t.TempDir(), "nvidia-smi")
|
||||||
|
script := "#!/bin/sh\ncat <<'EOF'\n" + smiSample + "EOF\n"
|
||||||
|
if err := os.WriteFile(stub, []byte(script), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return stub
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoverUsesSMIByDefault(t *testing.T) {
|
||||||
|
inv, err := Discover(t.Context(), Options{
|
||||||
|
NvidiaSMI: fakeSMI(t),
|
||||||
|
IBRoot: filepath.Join(t.TempDir(), "absent"),
|
||||||
|
}, slog.New(slog.DiscardHandler))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if inv.Source != "nvidia-smi" {
|
||||||
|
t.Errorf("source=%q", inv.Source)
|
||||||
|
}
|
||||||
|
if len(inv.GPUs) != 2 {
|
||||||
|
t.Errorf("got %d GPUs", len(inv.GPUs))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoverFallsBackWhenNVMLUnavailable(t *testing.T) {
|
||||||
|
// Asking for NVML in a build without it must still produce an inventory:
|
||||||
|
// the fallback is the whole point of the build tag.
|
||||||
|
inv, err := Discover(t.Context(), Options{
|
||||||
|
PreferNVML: true,
|
||||||
|
NvidiaSMI: fakeSMI(t),
|
||||||
|
IBRoot: filepath.Join(t.TempDir(), "absent"),
|
||||||
|
}, slog.New(slog.DiscardHandler))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(inv.GPUs) == 0 {
|
||||||
|
t.Fatal("no GPUs discovered through the fallback")
|
||||||
|
}
|
||||||
|
if NVMLSupported() {
|
||||||
|
// On an NVML build without a driver the NVML attempt fails and the
|
||||||
|
// fallback still has to work.
|
||||||
|
if inv.Source != "nvml" && inv.Source != "nvidia-smi" {
|
||||||
|
t.Errorf("source=%q", inv.Source)
|
||||||
|
}
|
||||||
|
} else if inv.Source != "nvidia-smi" {
|
||||||
|
t.Errorf("a build without NVML reported source=%q", inv.Source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoverSkipGPUs(t *testing.T) {
|
||||||
|
inv, err := Discover(t.Context(), Options{
|
||||||
|
SkipGPUs: true,
|
||||||
|
IBRoot: fakeSysfs(t),
|
||||||
|
}, slog.New(slog.DiscardHandler))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(inv.GPUs) != 0 {
|
||||||
|
t.Errorf("GPUs were discovered despite SkipGPUs")
|
||||||
|
}
|
||||||
|
if len(inv.HCAs) != 3 {
|
||||||
|
t.Errorf("got %d HCAs", len(inv.HCAs))
|
||||||
|
}
|
||||||
|
if inv.Source != "none" {
|
||||||
|
t.Errorf("source=%q", inv.Source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoverReportsGPUFailure(t *testing.T) {
|
||||||
|
// A node meant to have GPUs whose discovery fails must say so rather than
|
||||||
|
// reporting an empty, healthy-looking inventory.
|
||||||
|
_, err := Discover(context.Background(), Options{
|
||||||
|
NvidiaSMI: filepath.Join(t.TempDir(), "absent"),
|
||||||
|
IBRoot: filepath.Join(t.TempDir(), "absent"),
|
||||||
|
}, slog.New(slog.DiscardHandler))
|
||||||
|
if err == nil {
|
||||||
|
t.Error("a failed GPU discovery was reported as success")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNVMLSupportedMatchesBuild(t *testing.T) {
|
||||||
|
// Documents which backend this binary has, and keeps both files compiling.
|
||||||
|
t.Logf("NVML backend compiled in: %v", NVMLSupported())
|
||||||
|
if !NVMLSupported() {
|
||||||
|
if _, err := DiscoverGPUsNVML(t.Context()); err != ErrNoNVML {
|
||||||
|
t.Errorf("expected ErrNoNVML, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMemoryIsReportedEvenWhenGPUDiscoveryFails(t *testing.T) {
|
||||||
|
// A node with no nvidia-smi is exactly the node whose memory matters, since
|
||||||
|
// a CPU profile is what it will be asked to run. Reading memory after GPU
|
||||||
|
// discovery meant that node reported none: the GPU error returns early and
|
||||||
|
// takes a fact that has nothing to do with cards with it. Seen on node05,
|
||||||
|
// which has 32 GB and was reporting "-".
|
||||||
|
inv, err := Discover(t.Context(), Options{NvidiaSMI: "/nonexistent/nvidia-smi"}, nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("GPU discovery was expected to fail with no nvidia-smi")
|
||||||
|
}
|
||||||
|
// On Linux this is a real number; elsewhere there is no /proc/meminfo and 0
|
||||||
|
// is the honest answer. Either way it must not be lost to the GPU error.
|
||||||
|
if runtime.GOOS == "linux" && inv.MemoryTotalMB <= 0 {
|
||||||
|
t.Errorf("memory was not reported alongside a GPU failure: %d", inv.MemoryTotalMB)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FabricCheck is one declared fabric measured against what the node actually
|
||||||
|
// has.
|
||||||
|
type FabricCheck struct {
|
||||||
|
Fabric string
|
||||||
|
// Usable are the declared adapters that exist and have an active port.
|
||||||
|
Usable []string
|
||||||
|
// RateGbps is the slowest active port among them, because a collective
|
||||||
|
// runs at the speed of its slowest link.
|
||||||
|
RateGbps int
|
||||||
|
// Problems are the declared adapters that cannot carry traffic, and why.
|
||||||
|
Problems []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// OK reports whether this fabric can carry a collective at all.
|
||||||
|
func (c FabricCheck) OK() bool { return len(c.Usable) > 0 }
|
||||||
|
|
||||||
|
// CheckFabrics measures declared fabric membership against discovered adapters.
|
||||||
|
//
|
||||||
|
// Fabric membership has to be configuration — OpenSM's default subnet prefix is
|
||||||
|
// the same on every fabric, so a matching prefix proves nothing — but taking it
|
||||||
|
// entirely on trust was a mistake of a different kind. A typo, or a card that
|
||||||
|
// enumerated differently after a reboot, produced NCCL_IB_HCA naming an adapter
|
||||||
|
// that does not exist; and an adapter whose cable is out, or whose subnet
|
||||||
|
// manager is down, is present in sysfs and carries nothing.
|
||||||
|
//
|
||||||
|
// Either way the replica starts, passes its health checks, and hangs on its
|
||||||
|
// first all-reduce — which is the failure the whole fabric invariant exists to
|
||||||
|
// prevent, arriving through a door nobody was watching. Declared and real are
|
||||||
|
// now compared where both are visible, which is only here.
|
||||||
|
func CheckFabrics(declared map[string][]string, discovered []HCA) []FabricCheck {
|
||||||
|
byName := make(map[string]HCA, len(discovered))
|
||||||
|
for _, h := range discovered {
|
||||||
|
byName[h.Name] = h
|
||||||
|
}
|
||||||
|
|
||||||
|
names := make([]string, 0, len(declared))
|
||||||
|
for f := range declared {
|
||||||
|
names = append(names, f)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
|
||||||
|
out := make([]FabricCheck, 0, len(names))
|
||||||
|
for _, fabric := range names {
|
||||||
|
c := FabricCheck{Fabric: fabric}
|
||||||
|
for _, want := range declared[fabric] {
|
||||||
|
hca, found := byName[want]
|
||||||
|
if !found {
|
||||||
|
c.Problems = append(c.Problems, fmt.Sprintf(
|
||||||
|
"%s is not on this node (found: %s)", want, listNames(discovered)))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rate, active := activeRate(hca)
|
||||||
|
if !active {
|
||||||
|
c.Problems = append(c.Problems, fmt.Sprintf(
|
||||||
|
"%s has no active port (%s)", want, portStates(hca)))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c.Usable = append(c.Usable, want)
|
||||||
|
if c.RateGbps == 0 || rate < c.RateGbps {
|
||||||
|
c.RateGbps = rate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// activeRate returns the fastest active port's rate on an adapter.
|
||||||
|
//
|
||||||
|
// Only ACTIVE ports carry traffic. A port that is physically linked but has no
|
||||||
|
// subnet manager sits in INIT, looks healthy to anything counting adapters, and
|
||||||
|
// passes no data — which is precisely the case worth catching, because it
|
||||||
|
// survives a reboot and looks like a software problem.
|
||||||
|
func activeRate(h HCA) (int, bool) {
|
||||||
|
best, ok := 0, false
|
||||||
|
for _, p := range h.Ports {
|
||||||
|
if !strings.EqualFold(p.State, "ACTIVE") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ok = true
|
||||||
|
if p.RateGbps > best {
|
||||||
|
best = p.RateGbps
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return best, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
func portStates(h HCA) string {
|
||||||
|
if len(h.Ports) == 0 {
|
||||||
|
return "no ports"
|
||||||
|
}
|
||||||
|
var parts []string
|
||||||
|
for _, p := range h.Ports {
|
||||||
|
parts = append(parts, fmt.Sprintf("port %d %s/%s", p.Num, p.State, p.PhysState))
|
||||||
|
}
|
||||||
|
return strings.Join(parts, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func listNames(hcas []HCA) string {
|
||||||
|
if len(hcas) == 0 {
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for _, h := range hcas {
|
||||||
|
names = append(names, h.Name)
|
||||||
|
}
|
||||||
|
return strings.Join(names, ", ")
|
||||||
|
}
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func hca(name string, ports ...Port) HCA { return HCA{Name: name, Ports: ports} }
|
||||||
|
|
||||||
|
func active(rate int) Port {
|
||||||
|
return Port{Num: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: rate, LinkLayer: LinkInfiniBand}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestADeclaredAdapterThatIsNotThereIsReported(t *testing.T) {
|
||||||
|
// A typo, or a card that enumerated differently after a reboot. Taken on
|
||||||
|
// trust it becomes NCCL_IB_HCA naming an adapter that does not exist, and
|
||||||
|
// the replica hangs on its first all-reduce.
|
||||||
|
got := CheckFabrics(
|
||||||
|
map[string][]string{"ib-a": {"mlx5_0", "mlx5_9"}},
|
||||||
|
[]HCA{hca("mlx5_0", active(200))},
|
||||||
|
)
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("got %d checks", len(got))
|
||||||
|
}
|
||||||
|
c := got[0]
|
||||||
|
if len(c.Usable) != 1 || c.Usable[0] != "mlx5_0" {
|
||||||
|
t.Errorf("usable = %v, want only the adapter that exists", c.Usable)
|
||||||
|
}
|
||||||
|
if len(c.Problems) != 1 || !strings.Contains(c.Problems[0], "mlx5_9") {
|
||||||
|
t.Errorf("problems = %v", c.Problems)
|
||||||
|
}
|
||||||
|
// The message names what is actually present, because the operator is
|
||||||
|
// looking at a typo and cannot see the node.
|
||||||
|
if !strings.Contains(c.Problems[0], "mlx5_0") {
|
||||||
|
t.Errorf("the problem does not say what was found: %q", c.Problems[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAdapterWithNoActivePortCannotCarryTraffic(t *testing.T) {
|
||||||
|
// A port that is physically linked but has no subnet manager sits in INIT.
|
||||||
|
// It looks healthy to anything counting adapters and passes no data, which
|
||||||
|
// is the case worth catching: it survives a reboot and looks like software.
|
||||||
|
got := CheckFabrics(
|
||||||
|
map[string][]string{"ib-a": {"mlx5_0"}},
|
||||||
|
[]HCA{hca("mlx5_0", Port{Num: 1, State: "INIT", PhysState: "LinkUp", RateGbps: 200})},
|
||||||
|
)
|
||||||
|
c := got[0]
|
||||||
|
if c.OK() {
|
||||||
|
t.Error("a fabric with no active port was reported usable")
|
||||||
|
}
|
||||||
|
if len(c.Problems) != 1 || !strings.Contains(c.Problems[0], "INIT") {
|
||||||
|
t.Errorf("the problem does not say what state the port is in: %v", c.Problems)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheRateIsTheSlowestActiveLink(t *testing.T) {
|
||||||
|
// A collective runs at the speed of its slowest link, so reporting the
|
||||||
|
// fastest would have the scheduler believe a fabric is quicker than it is.
|
||||||
|
got := CheckFabrics(
|
||||||
|
map[string][]string{"ib-a": {"fast", "slow"}},
|
||||||
|
[]HCA{hca("fast", active(400)), hca("slow", active(100))},
|
||||||
|
)
|
||||||
|
if got[0].RateGbps != 100 {
|
||||||
|
t.Errorf("rate = %d, want 100", got[0].RateGbps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAdapterWithSeveralPortsUsesItsBestActiveOne(t *testing.T) {
|
||||||
|
// A dual-port card with one cable in is a perfectly good adapter, and
|
||||||
|
// judging it by the dead port would take a working fabric out of service.
|
||||||
|
got := CheckFabrics(
|
||||||
|
map[string][]string{"ib-a": {"mlx5_0"}},
|
||||||
|
[]HCA{hca("mlx5_0",
|
||||||
|
Port{Num: 1, State: "DOWN", PhysState: "Disabled"},
|
||||||
|
active(200),
|
||||||
|
)},
|
||||||
|
)
|
||||||
|
if !got[0].OK() {
|
||||||
|
t.Fatalf("a card with one live port was rejected: %+v", got[0])
|
||||||
|
}
|
||||||
|
if got[0].RateGbps != 200 {
|
||||||
|
t.Errorf("rate = %d, want the active port's 200", got[0].RateGbps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFabricWithNothingUsableIsNotOK(t *testing.T) {
|
||||||
|
got := CheckFabrics(
|
||||||
|
map[string][]string{"ib-dead": {"mlx5_0"}},
|
||||||
|
[]HCA{hca("mlx5_0", Port{Num: 1, State: "DOWN", PhysState: "Polling"})},
|
||||||
|
)
|
||||||
|
if got[0].OK() {
|
||||||
|
t.Error("a fabric whose only adapter is down was reported usable")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestANodeWithNoFabricsDeclaredProducesNoChecks(t *testing.T) {
|
||||||
|
// Every node without InfiniBand, which must stay entirely unaffected.
|
||||||
|
if got := CheckFabrics(nil, []HCA{hca("mlx5_0", active(200))}); len(got) != 0 {
|
||||||
|
t.Errorf("got %d checks for a node that declared none", len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChecksComeBackInAStableOrder(t *testing.T) {
|
||||||
|
// Map iteration would otherwise reorder log lines between restarts, which
|
||||||
|
// makes a diff of two registrations unreadable.
|
||||||
|
declared := map[string][]string{"ib-c": {"m"}, "ib-a": {"m"}, "ib-b": {"m"}}
|
||||||
|
got := CheckFabrics(declared, nil)
|
||||||
|
if len(got) != 3 {
|
||||||
|
t.Fatalf("got %d", len(got))
|
||||||
|
}
|
||||||
|
if got[0].Fabric != "ib-a" || got[1].Fabric != "ib-b" || got[2].Fabric != "ib-c" {
|
||||||
|
t.Errorf("order = %s, %s, %s", got[0].Fabric, got[1].Fabric, got[2].Fabric)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"regexp"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GPU is one accelerator on this node.
|
||||||
|
type GPU struct {
|
||||||
|
Index int `json:"index"`
|
||||||
|
UUID string `json:"uuid"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
// MemoryTotalMB is the total framebuffer size.
|
||||||
|
MemoryTotalMB int `json:"memory_total_mb"`
|
||||||
|
// MemoryUsedMB is what is already allocated. Non-zero on an idle node means
|
||||||
|
// something other than our engines is using the card (plan §5.1).
|
||||||
|
MemoryUsedMB int `json:"memory_used_mb"`
|
||||||
|
// UtilizationPercent is how busy the card was over the last sample period.
|
||||||
|
//
|
||||||
|
// The number people actually watch, and the one nothing collected: memory
|
||||||
|
// used says a model is loaded, and this says whether it is doing anything.
|
||||||
|
// A fleet of cards at ninety percent occupancy and two percent utilisation
|
||||||
|
// is a fleet holding models nobody is asking for, which is invisible
|
||||||
|
// without it.
|
||||||
|
UtilizationPercent int `json:"utilization_percent,omitempty"`
|
||||||
|
// ComputeCapability gates quantization choices: an FP8 profile must not
|
||||||
|
// land on a card below 8.9 (plan §6.2).
|
||||||
|
ComputeCapability string `json:"compute_capability"`
|
||||||
|
NUMANode int `json:"numa_node"`
|
||||||
|
PCIBusID string `json:"pci_bus_id,omitempty"`
|
||||||
|
DriverVersion string `json:"driver_version,omitempty"`
|
||||||
|
CUDAVersion string `json:"cuda_version,omitempty"`
|
||||||
|
// NVLinkPeers are the indices of GPUs reachable over NVLink. Within a node
|
||||||
|
// a TP group should stay inside one NVLink domain.
|
||||||
|
NVLinkPeers []int `json:"nvlink_peers,omitempty"`
|
||||||
|
// Health carries anything that should keep the GPU out of placement.
|
||||||
|
Health GPUHealth `json:"health"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// GPUHealth records conditions that make a GPU unsafe to schedule onto.
|
||||||
|
type GPUHealth struct {
|
||||||
|
// ECCErrors is the count of uncorrected ECC errors.
|
||||||
|
ECCErrors int `json:"ecc_errors"`
|
||||||
|
// TemperatureC is the current die temperature.
|
||||||
|
TemperatureC int `json:"temperature_c"`
|
||||||
|
// Problems lists human-readable reasons to avoid this GPU.
|
||||||
|
Problems []string `json:"problems,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// OK reports whether the GPU is fit for placement.
|
||||||
|
func (h GPUHealth) OK() bool { return len(h.Problems) == 0 }
|
||||||
|
|
||||||
|
// nonAlnum collapses everything that is not a letter or digit.
|
||||||
|
var nonAlnum = regexp.MustCompile(`[^a-z0-9]+`)
|
||||||
|
|
||||||
|
// Class derives the GPU class used for placement: same model, same memory.
|
||||||
|
//
|
||||||
|
// Two cards only belong in the same tensor-parallel group if they are the same
|
||||||
|
// class, so the derivation has to be stable across driver versions and naming
|
||||||
|
// quirks ("NVIDIA H100 80GB HBM3" and "H100-SXM5-80GB" are the same class).
|
||||||
|
func (g GPU) Class() string {
|
||||||
|
name := strings.ToLower(g.Name)
|
||||||
|
name = strings.TrimPrefix(name, "nvidia ")
|
||||||
|
name = strings.TrimPrefix(name, "tesla ")
|
||||||
|
|
||||||
|
// Keep the model token: the first word that contains a digit, which is how
|
||||||
|
// NVIDIA names every accelerator (h100, a100, l40s, rtx4090).
|
||||||
|
var model string
|
||||||
|
for _, tok := range nonAlnum.Split(name, -1) {
|
||||||
|
if tok == "" || !strings.ContainsFunc(tok, func(r rune) bool { return r >= '0' && r <= '9' }) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Skip memory and form-factor tokens, which are not the model.
|
||||||
|
if strings.HasSuffix(tok, "gb") || tok == "80gb" || tok == "sxm4" || tok == "sxm5" || tok == "pcie" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
model = tok
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if model == "" {
|
||||||
|
model = nonAlnum.ReplaceAllString(name, "-")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Round to the nearest GiB: vendors report 81559 MiB for an "80GB" card.
|
||||||
|
gb := int(math.Round(float64(g.MemoryTotalMB) / 1024))
|
||||||
|
return fmt.Sprintf("%s-%dg", model, gb)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalJSON adds the derived class, so a consumer of the JSON does not have
|
||||||
|
// to reimplement the derivation to know which GPUs are interchangeable.
|
||||||
|
func (g GPU) MarshalJSON() ([]byte, error) {
|
||||||
|
type alias GPU // prevents recursing back into this method
|
||||||
|
return json.Marshal(struct {
|
||||||
|
alias
|
||||||
|
Class string `json:"class"`
|
||||||
|
}{alias(g), g.Class()})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Inventory is everything discovery found on this node.
|
||||||
|
type Inventory struct {
|
||||||
|
GPUs []GPU `json:"gpus"`
|
||||||
|
HCAs []HCA `json:"hcas"`
|
||||||
|
// Source names the backend that produced the GPU list, so a consumer can
|
||||||
|
// tell "no GPUs" from "discovery did not work".
|
||||||
|
Source string `json:"source"`
|
||||||
|
// MemoryTotalMB is the machine's RAM. Zero means unknown, which is what an
|
||||||
|
// agent older than this field reports, and is treated as no constraint.
|
||||||
|
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
|
||||||
|
// Weights are the model files this node can see. Empty means it was not
|
||||||
|
// asked to look.
|
||||||
|
Weights []LocalWeights `json:"weights,omitempty"`
|
||||||
|
// Storage is the disk behind those weights, one entry per filesystem.
|
||||||
|
// Empty means the same thing it means for Weights: not asked.
|
||||||
|
Storage []Storage `json:"storage,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Classes returns the distinct GPU classes present, sorted.
|
||||||
|
func (inv Inventory) Classes() []string {
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
for _, g := range inv.GPUs {
|
||||||
|
seen[g.Class()] = struct{}{}
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(seen))
|
||||||
|
for c := range seen {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
slices.Sort(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseCSVInt reads an integer from nvidia-smi's CSV output, tolerating the
|
||||||
|
// "[N/A]" and "Not Supported" values it emits for unsupported queries.
|
||||||
|
func parseCSVInt(s string) (int, bool) {
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
s = strings.TrimSuffix(s, " MiB")
|
||||||
|
s = strings.TrimSuffix(s, " W")
|
||||||
|
if s == "" || strings.HasPrefix(s, "[") || strings.EqualFold(s, "Not Supported") {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
v, err := strconv.Atoi(s)
|
||||||
|
if err != nil {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
return v, true
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
//go:build !nvml
|
||||||
|
|
||||||
|
package discovery
|
||||||
|
|
||||||
|
import "context"
|
||||||
|
|
||||||
|
// nvmlAvailable reports that this build cannot use NVML.
|
||||||
|
const nvmlAvailable = false
|
||||||
|
|
||||||
|
// DiscoverGPUsNVML is unavailable in builds without the nvml tag.
|
||||||
|
func DiscoverGPUsNVML(context.Context) ([]GPU, error) { return nil, ErrNoNVML }
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
//go:build nvml
|
||||||
|
|
||||||
|
// This file is the NVML-backed GPU discovery path, built only with the `nvml`
|
||||||
|
// tag because the bindings require cgo. Without the tag the agent falls back to
|
||||||
|
// nvidia-smi, which keeps the server side — and development machines without a
|
||||||
|
// C toolchain or a driver — building cleanly (plan §3.2).
|
||||||
|
//
|
||||||
|
// What NVML adds over nvidia-smi is NVLink topology. Nothing else exposes which
|
||||||
|
// GPUs share an NVLink domain, and that is what keeps a tensor-parallel group
|
||||||
|
// off the slow path inside a node (plan §6.2, constraint 3).
|
||||||
|
|
||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/NVIDIA/go-nvml/pkg/nvml"
|
||||||
|
)
|
||||||
|
|
||||||
|
// nvmlAvailable reports that this build can use NVML.
|
||||||
|
const nvmlAvailable = true
|
||||||
|
|
||||||
|
// DiscoverGPUsNVML enumerates GPUs through NVML.
|
||||||
|
//
|
||||||
|
// It returns an error rather than falling back on its own: the caller decides
|
||||||
|
// whether to degrade to nvidia-smi, because on a node that is supposed to have
|
||||||
|
// working NVML a silent downgrade hides a broken driver installation.
|
||||||
|
func DiscoverGPUsNVML(_ context.Context) ([]GPU, error) {
|
||||||
|
if ret := nvml.Init(); ret != nvml.SUCCESS {
|
||||||
|
return nil, fmt.Errorf("nvml init: %s", nvml.ErrorString(ret))
|
||||||
|
}
|
||||||
|
defer func() { _ = nvml.Shutdown() }()
|
||||||
|
|
||||||
|
driver, _ := nvml.SystemGetDriverVersion()
|
||||||
|
cuda := ""
|
||||||
|
if v, ret := nvml.SystemGetCudaDriverVersion(); ret == nvml.SUCCESS {
|
||||||
|
// NVML reports 12040 for 12.4.
|
||||||
|
cuda = fmt.Sprintf("%d.%d", v/1000, (v%1000)/10)
|
||||||
|
}
|
||||||
|
|
||||||
|
count, ret := nvml.DeviceGetCount()
|
||||||
|
if ret != nvml.SUCCESS {
|
||||||
|
return nil, fmt.Errorf("nvml device count: %s", nvml.ErrorString(ret))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bus id to index, so NVLink peers can be reported as indices. NVML gives
|
||||||
|
// the remote end of a link as a PCI address, which is not what placement
|
||||||
|
// works in.
|
||||||
|
byBusID := make(map[string]int, count)
|
||||||
|
gpus := make([]GPU, 0, count)
|
||||||
|
|
||||||
|
for i := range count {
|
||||||
|
dev, ret := nvml.DeviceGetHandleByIndex(i)
|
||||||
|
if ret != nvml.SUCCESS {
|
||||||
|
return nil, fmt.Errorf("nvml device %d: %s", i, nvml.ErrorString(ret))
|
||||||
|
}
|
||||||
|
|
||||||
|
g := GPU{Index: i, NUMANode: -1, DriverVersion: driver, CUDAVersion: cuda}
|
||||||
|
g.UUID, _ = dev.GetUUID()
|
||||||
|
g.Name, _ = dev.GetName()
|
||||||
|
|
||||||
|
if mem, ret := dev.GetMemoryInfo(); ret == nvml.SUCCESS {
|
||||||
|
g.MemoryTotalMB = int(mem.Total / (1 << 20))
|
||||||
|
g.MemoryUsedMB = int(mem.Used / (1 << 20))
|
||||||
|
}
|
||||||
|
if major, minor, ret := dev.GetCudaComputeCapability(); ret == nvml.SUCCESS {
|
||||||
|
g.ComputeCapability = fmt.Sprintf("%d.%d", major, minor)
|
||||||
|
}
|
||||||
|
if pci, ret := dev.GetPciInfo(); ret == nvml.SUCCESS {
|
||||||
|
g.PCIBusID = busIDString(pci.BusId[:])
|
||||||
|
byBusID[normalizeBusID(g.PCIBusID)] = i
|
||||||
|
}
|
||||||
|
if t, ret := dev.GetTemperature(nvml.TEMPERATURE_GPU); ret == nvml.SUCCESS {
|
||||||
|
g.Health.TemperatureC = int(t)
|
||||||
|
}
|
||||||
|
// Volatile counters reset on driver reload, which is what we want: an
|
||||||
|
// error from three months and one reboot ago is not a reason to refuse
|
||||||
|
// a card today.
|
||||||
|
if n, ret := dev.GetTotalEccErrors(nvml.MEMORY_ERROR_TYPE_UNCORRECTED, nvml.VOLATILE_ECC); ret == nvml.SUCCESS {
|
||||||
|
g.Health.ECCErrors = int(n)
|
||||||
|
}
|
||||||
|
|
||||||
|
assessHealth(&g)
|
||||||
|
gpus = append(gpus, g)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A second pass, because a link's remote end may be a GPU enumerated later.
|
||||||
|
for i := range gpus {
|
||||||
|
dev, ret := nvml.DeviceGetHandleByIndex(gpus[i].Index)
|
||||||
|
if ret != nvml.SUCCESS {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
gpus[i].NVLinkPeers = nvlinkPeers(dev, byBusID, gpus[i].Index)
|
||||||
|
}
|
||||||
|
return gpus, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// nvlinkPeers returns the indices of GPUs reachable from dev over NVLink.
|
||||||
|
func nvlinkPeers(dev nvml.Device, byBusID map[string]int, self int) []int {
|
||||||
|
seen := map[int]struct{}{}
|
||||||
|
var out []int
|
||||||
|
|
||||||
|
for link := range nvml.NVLINK_MAX_LINKS {
|
||||||
|
pci, ret := dev.GetNvLinkRemotePciInfo(link)
|
||||||
|
if ret != nvml.SUCCESS {
|
||||||
|
// Unpopulated links return an error; that is the normal case for
|
||||||
|
// most of the 36 possible links, so it is not worth reporting.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
peer, ok := byBusID[normalizeBusID(busIDString(pci.BusId[:]))]
|
||||||
|
if !ok || peer == self {
|
||||||
|
// The remote end can be an NVSwitch rather than a GPU, in which
|
||||||
|
// case it is not in the map.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, dup := seen[peer]; dup {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[peer] = struct{}{}
|
||||||
|
out = append(out, peer)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// busIDString turns NVML's fixed-size C char array into a Go string.
|
||||||
|
func busIDString(b []int8) string {
|
||||||
|
buf := make([]byte, 0, len(b))
|
||||||
|
for _, c := range b {
|
||||||
|
if c == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
buf = append(buf, byte(c))
|
||||||
|
}
|
||||||
|
return string(buf)
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizeBusID makes bus ids comparable: NVML reports them uppercase from one
|
||||||
|
// call and lowercase from another, and sometimes with a shortened domain.
|
||||||
|
func normalizeBusID(s string) string {
|
||||||
|
s = strings.ToLower(strings.TrimSpace(s))
|
||||||
|
// "0000:1b:00.0" and "00000000:1B:00.0" are the same device.
|
||||||
|
if i := strings.Index(s, ":"); i > 0 {
|
||||||
|
s = strings.TrimLeft(s[:i], "0") + s[i:]
|
||||||
|
}
|
||||||
|
return strings.TrimPrefix(s, ":")
|
||||||
|
}
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/csv"
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// smiQueryFields are the columns requested from nvidia-smi, in order.
|
||||||
|
var smiQueryFields = []string{
|
||||||
|
"index",
|
||||||
|
"uuid",
|
||||||
|
"name",
|
||||||
|
"memory.total",
|
||||||
|
"memory.used",
|
||||||
|
"compute_cap",
|
||||||
|
"pci.bus_id",
|
||||||
|
"driver_version",
|
||||||
|
"temperature.gpu",
|
||||||
|
"ecc.errors.uncorrected.volatile.total",
|
||||||
|
// Last, and optional. An nvidia-smi too old to know this field returns a
|
||||||
|
// shorter row, and every column before it still lines up — which is why it
|
||||||
|
// goes on the end rather than anywhere else, and why the parser requires
|
||||||
|
// only the mandatory prefix.
|
||||||
|
"utilization.gpu",
|
||||||
|
}
|
||||||
|
|
||||||
|
// mandatorySMIFields is how many columns a row must have to be usable.
|
||||||
|
//
|
||||||
|
// Everything before utilization.gpu. A driver that does not know the last field
|
||||||
|
// gives a short row, and rejecting those would make an llmgw upgrade silently
|
||||||
|
// stop discovering GPUs on an older node — a far worse outcome than not knowing
|
||||||
|
// how busy a card is.
|
||||||
|
const mandatorySMIFields = 10
|
||||||
|
|
||||||
|
// DiscoverGPUsSMI enumerates GPUs by shelling out to nvidia-smi.
|
||||||
|
//
|
||||||
|
// This is the cgo-free path (plan §3.2): the NVML bindings need cgo, which
|
||||||
|
// would force the whole binary — server included — to be built with a C
|
||||||
|
// toolchain. nvidia-smi gives everything except NVLink topology, which the
|
||||||
|
// NVML build adds.
|
||||||
|
func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) {
|
||||||
|
if binary == "" {
|
||||||
|
binary = "nvidia-smi"
|
||||||
|
}
|
||||||
|
out, err := exec.CommandContext(ctx, binary,
|
||||||
|
"--query-gpu="+strings.Join(smiQueryFields, ","),
|
||||||
|
"--format=csv,noheader,nounits",
|
||||||
|
).Output()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("running %s: %w", binary, err)
|
||||||
|
}
|
||||||
|
return parseSMI(string(out))
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseSMI(out string) ([]GPU, error) {
|
||||||
|
r := csv.NewReader(strings.NewReader(out))
|
||||||
|
r.TrimLeadingSpace = true
|
||||||
|
r.FieldsPerRecord = -1 // tolerate driver versions with extra columns
|
||||||
|
|
||||||
|
rows, err := r.ReadAll()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parsing nvidia-smi output: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
gpus := make([]GPU, 0, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
if len(row) < mandatorySMIFields {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
idx, err := strconv.Atoi(strings.TrimSpace(row[0]))
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
g := GPU{
|
||||||
|
Index: idx,
|
||||||
|
UUID: strings.TrimSpace(row[1]),
|
||||||
|
Name: strings.TrimSpace(row[2]),
|
||||||
|
ComputeCapability: strings.TrimSpace(row[5]),
|
||||||
|
PCIBusID: strings.TrimSpace(row[6]),
|
||||||
|
DriverVersion: strings.TrimSpace(row[7]),
|
||||||
|
NUMANode: -1,
|
||||||
|
}
|
||||||
|
g.MemoryTotalMB, _ = parseCSVInt(row[3])
|
||||||
|
g.MemoryUsedMB, _ = parseCSVInt(row[4])
|
||||||
|
g.Health.TemperatureC, _ = parseCSVInt(row[8])
|
||||||
|
g.Health.ECCErrors, _ = parseCSVInt(row[9])
|
||||||
|
if len(row) > 10 {
|
||||||
|
g.UtilizationPercent, _ = parseCSVInt(row[10])
|
||||||
|
}
|
||||||
|
|
||||||
|
assessHealth(&g)
|
||||||
|
gpus = append(gpus, g)
|
||||||
|
}
|
||||||
|
return gpus, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// residentMemoryToleranceMB is how much allocated memory is written off as
|
||||||
|
// driver overhead rather than a foreign process.
|
||||||
|
const residentMemoryToleranceMB = 512
|
||||||
|
|
||||||
|
// assessHealth flags conditions that should keep a GPU out of placement.
|
||||||
|
func assessHealth(g *GPU) {
|
||||||
|
if g.Health.ECCErrors > 0 {
|
||||||
|
g.Health.Problems = append(g.Health.Problems,
|
||||||
|
fmt.Sprintf("%d uncorrected ECC errors", g.Health.ECCErrors))
|
||||||
|
}
|
||||||
|
// Memory already in use means something else is on the card. Starting an
|
||||||
|
// engine there would either fail to allocate or quietly evict the other
|
||||||
|
// process's throughput.
|
||||||
|
if g.MemoryUsedMB > residentMemoryToleranceMB {
|
||||||
|
g.Health.Problems = append(g.Health.Problems,
|
||||||
|
fmt.Sprintf("%d MiB already allocated by another process", g.MemoryUsedMB))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,179 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
const smiSample = `0, GPU-11111111-2222-3333-4444-555555555555, NVIDIA H100 80GB HBM3, 81559, 4, 9.0, 00000000:1B:00.0, 550.54.15, 34, 0
|
||||||
|
1, GPU-66666666-7777-8888-9999-000000000000, NVIDIA H100 80GB HBM3, 81559, 0, 9.0, 00000000:1C:00.0, 550.54.15, 36, 0
|
||||||
|
`
|
||||||
|
|
||||||
|
func TestParseSMI(t *testing.T) {
|
||||||
|
gpus, err := parseSMI(smiSample)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(gpus) != 2 {
|
||||||
|
t.Fatalf("expected 2 GPUs, got %d", len(gpus))
|
||||||
|
}
|
||||||
|
|
||||||
|
g := gpus[0]
|
||||||
|
if g.Index != 0 || g.UUID != "GPU-11111111-2222-3333-4444-555555555555" {
|
||||||
|
t.Errorf("identity wrong: %+v", g)
|
||||||
|
}
|
||||||
|
if g.MemoryTotalMB != 81559 {
|
||||||
|
t.Errorf("MemoryTotalMB=%d", g.MemoryTotalMB)
|
||||||
|
}
|
||||||
|
if g.ComputeCapability != "9.0" {
|
||||||
|
t.Errorf("ComputeCapability=%q; FP8 placement depends on it", g.ComputeCapability)
|
||||||
|
}
|
||||||
|
if g.DriverVersion != "550.54.15" {
|
||||||
|
t.Errorf("DriverVersion=%q", g.DriverVersion)
|
||||||
|
}
|
||||||
|
if !g.Health.OK() {
|
||||||
|
t.Errorf("a healthy GPU was flagged: %v", g.Health.Problems)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseSMIUnsupportedFields(t *testing.T) {
|
||||||
|
// Consumer cards report [N/A] for ECC; that must not fail the whole parse.
|
||||||
|
const row = `0, GPU-abc, NVIDIA GeForce RTX 4090, 24564, 0, 8.9, 00000000:01:00.0, 550.54.15, 45, [N/A]
|
||||||
|
`
|
||||||
|
gpus, err := parseSMI(row)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(gpus) != 1 {
|
||||||
|
t.Fatalf("expected 1 GPU, got %d", len(gpus))
|
||||||
|
}
|
||||||
|
if gpus[0].Health.ECCErrors != 0 || !gpus[0].Health.OK() {
|
||||||
|
t.Errorf("[N/A] ECC was misread: %+v", gpus[0].Health)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHealthFlagsForeignMemory(t *testing.T) {
|
||||||
|
// Memory in use on an otherwise idle card means another process is there.
|
||||||
|
const row = `0, GPU-abc, NVIDIA H100 80GB HBM3, 81559, 40000, 9.0, 00000000:1B:00.0, 550.54.15, 34, 0
|
||||||
|
`
|
||||||
|
gpus, _ := parseSMI(row)
|
||||||
|
if gpus[0].Health.OK() {
|
||||||
|
t.Error("40 GiB allocated by another process was not flagged")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHealthFlagsECC(t *testing.T) {
|
||||||
|
const row = `0, GPU-abc, NVIDIA H100 80GB HBM3, 81559, 0, 9.0, 00000000:1B:00.0, 550.54.15, 34, 3
|
||||||
|
`
|
||||||
|
gpus, _ := parseSMI(row)
|
||||||
|
if gpus[0].Health.OK() {
|
||||||
|
t.Error("uncorrected ECC errors were not flagged")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGPUClass(t *testing.T) {
|
||||||
|
// The same card must derive the same class however the driver names it,
|
||||||
|
// because a tensor-parallel group may not mix classes.
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
memMB int
|
||||||
|
expect string
|
||||||
|
}{
|
||||||
|
{"NVIDIA H100 80GB HBM3", 81559, "h100-80g"},
|
||||||
|
{"NVIDIA H100-SXM5-80GB", 81559, "h100-80g"},
|
||||||
|
{"Tesla A100-SXM4-40GB", 40960, "a100-40g"},
|
||||||
|
{"NVIDIA A100 80GB PCIe", 81920, "a100-80g"},
|
||||||
|
// Sizes come from what the driver reports, in GiB. That does not always
|
||||||
|
// match the marketing name: an L40S is sold as 48GB but reports 44.99
|
||||||
|
// GiB, because the marketing figure is decimal GB. Being consistent
|
||||||
|
// matters more than matching the box, since the number is what decides
|
||||||
|
// whether a model fits.
|
||||||
|
{"NVIDIA L40S", 46068, "l40s-45g"},
|
||||||
|
// The model token is what distinguishes a 4090 from a 4080, so "rtx"
|
||||||
|
// is not enough on its own.
|
||||||
|
{"NVIDIA GeForce RTX 4090", 24564, "4090-24g"},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
got := GPU{Name: c.name, MemoryTotalMB: c.memMB}.Class()
|
||||||
|
if got != c.expect {
|
||||||
|
t.Errorf("%q (%d MiB) -> %q, want %q", c.name, c.memMB, got, c.expect)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInventoryClasses(t *testing.T) {
|
||||||
|
inv := Inventory{GPUs: []GPU{
|
||||||
|
{Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559},
|
||||||
|
{Name: "NVIDIA H100-SXM5-80GB", MemoryTotalMB: 81559},
|
||||||
|
{Name: "Tesla A100-SXM4-40GB", MemoryTotalMB: 40960},
|
||||||
|
}}
|
||||||
|
got := inv.Classes()
|
||||||
|
if len(got) != 2 || got[0] != "a100-40g" || got[1] != "h100-80g" {
|
||||||
|
t.Errorf("classes=%v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDiscoverGPUsSMI runs the real code path against a stub binary, so the
|
||||||
|
// command construction is covered too, not just the parser.
|
||||||
|
func TestDiscoverGPUsSMI(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
stub := filepath.Join(dir, "nvidia-smi")
|
||||||
|
script := "#!/bin/sh\ncat <<'EOF'\n" + smiSample + "EOF\n"
|
||||||
|
if err := os.WriteFile(stub, []byte(script), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
gpus, err := DiscoverGPUsSMI(t.Context(), stub)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(gpus) != 2 {
|
||||||
|
t.Fatalf("expected 2 GPUs, got %d", len(gpus))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoverGPUsSMIMissingBinary(t *testing.T) {
|
||||||
|
// A node with no NVIDIA driver must produce a clear error rather than a
|
||||||
|
// panic or an empty success.
|
||||||
|
_, err := DiscoverGPUsSMI(context.Background(), filepath.Join(t.TempDir(), "absent"))
|
||||||
|
if err == nil {
|
||||||
|
t.Error("expected an error when nvidia-smi is missing")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnOlderDriverWithoutUtilisationStillDiscovers(t *testing.T) {
|
||||||
|
// An nvidia-smi too old for utilization.gpu returns a shorter row. Rejecting
|
||||||
|
// those would make an llmgw upgrade silently stop discovering GPUs on an
|
||||||
|
// older node, which is far worse than not knowing how busy a card is — and
|
||||||
|
// is what the first version of this change did.
|
||||||
|
old := "0, GPU-aaa, NVIDIA H100 80GB HBM3, 81559, 1024, 9.0, 0000:01:00.0, 550.54, 41, 0"
|
||||||
|
gpus, err := parseSMI(old)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(gpus) != 1 {
|
||||||
|
t.Fatalf("a row without utilisation produced %d GPUs", len(gpus))
|
||||||
|
}
|
||||||
|
if gpus[0].MemoryTotalMB != 81559 || gpus[0].Health.TemperatureC != 41 {
|
||||||
|
t.Errorf("columns shifted: %+v", gpus[0])
|
||||||
|
}
|
||||||
|
if gpus[0].UtilizationPercent != 0 {
|
||||||
|
t.Errorf("utilisation = %d, want 0 when unreported", gpus[0].UtilizationPercent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUtilisationIsReadWhenPresent(t *testing.T) {
|
||||||
|
row := "0, GPU-aaa, NVIDIA H100 80GB HBM3, 81559, 1024, 9.0, 0000:01:00.0, 550.54, 41, 0, 87"
|
||||||
|
gpus, err := parseSMI(row)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(gpus) != 1 {
|
||||||
|
t.Fatalf("got %d GPUs", len(gpus))
|
||||||
|
}
|
||||||
|
if gpus[0].UtilizationPercent != 87 {
|
||||||
|
t.Errorf("utilisation = %d, want 87", gpus[0].UtilizationPercent)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,203 @@
|
|||||||
|
// Package discovery inspects a node's GPUs, InfiniBand adapters and the
|
||||||
|
// topology between them (plan §5.1 and §5.2).
|
||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SysfsRoot is the default place InfiniBand devices appear.
|
||||||
|
const SysfsRoot = "/sys/class/infiniband"
|
||||||
|
|
||||||
|
// LinkLayer distinguishes real InfiniBand from RoCE.
|
||||||
|
type LinkLayer string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// LinkInfiniBand is native InfiniBand.
|
||||||
|
LinkInfiniBand LinkLayer = "InfiniBand"
|
||||||
|
// LinkEthernet is RoCE — Ethernet carrying RDMA.
|
||||||
|
LinkEthernet LinkLayer = "Ethernet"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Port is one port of an HCA.
|
||||||
|
type Port struct {
|
||||||
|
Num int `json:"num"`
|
||||||
|
// State is the port's logical state; only ACTIVE ports carry traffic.
|
||||||
|
State string `json:"state"`
|
||||||
|
// PhysState is the physical link state, useful for telling "cable unplugged"
|
||||||
|
// from "no subnet manager".
|
||||||
|
PhysState string `json:"phys_state"`
|
||||||
|
// RateGbps is the negotiated rate.
|
||||||
|
RateGbps int `json:"rate_gbps"`
|
||||||
|
LinkLayer LinkLayer `json:"link_layer"`
|
||||||
|
// GIDPrefix is the subnet prefix from gids/0, normalized to lowercase hex
|
||||||
|
// groups. It is the only automatic fabric hint available, and an unreliable
|
||||||
|
// one: OpenSM's default is fe80:: everywhere (plan §5.2).
|
||||||
|
GIDPrefix string `json:"gid_prefix"`
|
||||||
|
LID int `json:"lid"`
|
||||||
|
SMLID int `json:"sm_lid"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Active reports whether the port can carry traffic.
|
||||||
|
func (p Port) Active() bool { return strings.EqualFold(p.State, "ACTIVE") }
|
||||||
|
|
||||||
|
// HCA is one host channel adapter.
|
||||||
|
type HCA struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Ports []Port `json:"ports"`
|
||||||
|
PCIAddr string `json:"pci_addr,omitempty"`
|
||||||
|
NUMANode int `json:"numa_node"`
|
||||||
|
// FirmwareVersion is reported by the driver; worth capturing because
|
||||||
|
// NCCL behaviour differs between firmware revisions.
|
||||||
|
FirmwareVersion string `json:"firmware_version,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ActivePorts returns the ports that can carry traffic.
|
||||||
|
func (h HCA) ActivePorts() []Port {
|
||||||
|
var out []Port
|
||||||
|
for _, p := range h.Ports {
|
||||||
|
if p.Active() {
|
||||||
|
out = append(out, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// DiscoverIB enumerates the InfiniBand adapters under root.
|
||||||
|
//
|
||||||
|
// A missing root is not an error: plenty of nodes have no IB at all, and the
|
||||||
|
// scheduler treats them as single-node islands (plan §2).
|
||||||
|
func DiscoverIB(root string) ([]HCA, error) {
|
||||||
|
if root == "" {
|
||||||
|
root = SysfsRoot
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(root)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("reading %s: %w", root, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var out []HCA
|
||||||
|
for _, e := range entries {
|
||||||
|
hca, err := readHCA(filepath.Join(root, e.Name()), e.Name())
|
||||||
|
if err != nil {
|
||||||
|
// One unreadable adapter must not hide the rest; the node may still
|
||||||
|
// be usable through its other HCAs.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, hca)
|
||||||
|
}
|
||||||
|
slices.SortFunc(out, func(a, b HCA) int { return strings.Compare(a.Name, b.Name) })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readHCA(dir, name string) (HCA, error) {
|
||||||
|
h := HCA{Name: name, NUMANode: -1}
|
||||||
|
h.FirmwareVersion = readTrimmed(filepath.Join(dir, "fw_ver"))
|
||||||
|
|
||||||
|
// device is a symlink into the PCI tree; its target's base name is the
|
||||||
|
// bus address, and numa_node lives beside it.
|
||||||
|
if target, err := filepath.EvalSymlinks(filepath.Join(dir, "device")); err == nil {
|
||||||
|
h.PCIAddr = filepath.Base(target)
|
||||||
|
if n, err := strconv.Atoi(readTrimmed(filepath.Join(target, "numa_node"))); err == nil {
|
||||||
|
h.NUMANode = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
portDirs, err := os.ReadDir(filepath.Join(dir, "ports"))
|
||||||
|
if err != nil {
|
||||||
|
return h, fmt.Errorf("%s: no ports: %w", name, err)
|
||||||
|
}
|
||||||
|
for _, pd := range portDirs {
|
||||||
|
num, err := strconv.Atoi(pd.Name())
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
h.Ports = append(h.Ports, readPort(filepath.Join(dir, "ports", pd.Name()), num))
|
||||||
|
}
|
||||||
|
slices.SortFunc(h.Ports, func(a, b Port) int { return a.Num - b.Num })
|
||||||
|
return h, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readPort(dir string, num int) Port {
|
||||||
|
p := Port{
|
||||||
|
Num: num,
|
||||||
|
State: parseEnumeratedState(readTrimmed(filepath.Join(dir, "state"))),
|
||||||
|
PhysState: parseEnumeratedState(readTrimmed(filepath.Join(dir, "phys_state"))),
|
||||||
|
RateGbps: parseRate(readTrimmed(filepath.Join(dir, "rate"))),
|
||||||
|
LinkLayer: LinkLayer(readTrimmed(filepath.Join(dir, "link_layer"))),
|
||||||
|
GIDPrefix: parseGIDPrefix(readTrimmed(filepath.Join(dir, "gids", "0"))),
|
||||||
|
LID: parseMaybeHex(readTrimmed(filepath.Join(dir, "lid"))),
|
||||||
|
SMLID: parseMaybeHex(readTrimmed(filepath.Join(dir, "sm_lid"))),
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseEnumeratedState turns sysfs's "4: ACTIVE" into "ACTIVE".
|
||||||
|
func parseEnumeratedState(s string) string {
|
||||||
|
if _, rest, found := strings.Cut(s, ":"); found {
|
||||||
|
return strings.TrimSpace(rest)
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseRate turns "100 Gb/sec (4X EDR)" into 100.
|
||||||
|
func parseRate(s string) int {
|
||||||
|
f := strings.Fields(s)
|
||||||
|
if len(f) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
// The rate can be fractional ("2.5 Gb/sec") on old hardware; truncate.
|
||||||
|
v, err := strconv.ParseFloat(f[0], 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return int(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseGIDPrefix keeps the first four groups of a GID, which is the subnet
|
||||||
|
// prefix; the remaining four are the interface id.
|
||||||
|
func parseGIDPrefix(gid string) string {
|
||||||
|
parts := strings.Split(strings.ToLower(gid), ":")
|
||||||
|
if len(parts) < 4 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.Join(parts[:4], ":")
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseMaybeHex accepts both "0x1a" and "26"; sysfs is inconsistent between
|
||||||
|
// drivers and kernel versions.
|
||||||
|
func parseMaybeHex(s string) int {
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
if s == "" {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if after, ok := strings.CutPrefix(strings.ToLower(s), "0x"); ok {
|
||||||
|
v, err := strconv.ParseInt(after, 16, 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return int(v)
|
||||||
|
}
|
||||||
|
v, err := strconv.Atoi(s)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
func readTrimmed(path string) string {
|
||||||
|
b, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(string(b))
|
||||||
|
}
|
||||||
@@ -0,0 +1,216 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeSysfs builds an InfiniBand tree shaped like the real one.
|
||||||
|
func fakeSysfs(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
|
||||||
|
write := func(path, content string) {
|
||||||
|
t.Helper()
|
||||||
|
full := filepath.Join(root, path)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(full, []byte(content+"\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// mlx5_0: one active IB port.
|
||||||
|
write("ib/mlx5_0/fw_ver", "20.35.1012")
|
||||||
|
write("ib/mlx5_0/ports/1/state", "4: ACTIVE")
|
||||||
|
write("ib/mlx5_0/ports/1/phys_state", "5: LinkUp")
|
||||||
|
write("ib/mlx5_0/ports/1/rate", "200 Gb/sec (4X HDR)")
|
||||||
|
write("ib/mlx5_0/ports/1/link_layer", "InfiniBand")
|
||||||
|
write("ib/mlx5_0/ports/1/gids/0", "FEC0:0000:0000:000A:0002:C903:00A0:1234")
|
||||||
|
write("ib/mlx5_0/ports/1/lid", "0x1a")
|
||||||
|
write("ib/mlx5_0/ports/1/sm_lid", "1")
|
||||||
|
|
||||||
|
// mlx5_1: two ports, the second one down.
|
||||||
|
write("ib/mlx5_1/ports/1/state", "4: ACTIVE")
|
||||||
|
write("ib/mlx5_1/ports/1/rate", "200 Gb/sec (4X HDR)")
|
||||||
|
write("ib/mlx5_1/ports/1/link_layer", "InfiniBand")
|
||||||
|
write("ib/mlx5_1/ports/1/gids/0", "fec0:0000:0000:000a:0002:c903:00a0:5678")
|
||||||
|
write("ib/mlx5_1/ports/2/state", "1: DOWN")
|
||||||
|
write("ib/mlx5_1/ports/2/phys_state", "3: Disabled")
|
||||||
|
write("ib/mlx5_1/ports/2/rate", "0")
|
||||||
|
write("ib/mlx5_1/ports/2/link_layer", "InfiniBand")
|
||||||
|
|
||||||
|
// mlx5_2: RoCE rather than native IB.
|
||||||
|
write("ib/mlx5_2/ports/1/state", "4: ACTIVE")
|
||||||
|
write("ib/mlx5_2/ports/1/rate", "100 Gb/sec (4X EDR)")
|
||||||
|
write("ib/mlx5_2/ports/1/link_layer", "Ethernet")
|
||||||
|
write("ib/mlx5_2/ports/1/gids/0", "0000:0000:0000:0000:0000:ffff:0a00:0001")
|
||||||
|
|
||||||
|
// A PCI device with a NUMA node, linked the way the driver does it.
|
||||||
|
pci := filepath.Join(root, "pci", "0000:0b:00.0")
|
||||||
|
if err := os.MkdirAll(pci, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(pci, "numa_node"), []byte("1\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(pci, filepath.Join(root, "ib", "mlx5_0", "device")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return filepath.Join(root, "ib")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiscoverIB(t *testing.T) {
|
||||||
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(hcas) != 3 {
|
||||||
|
t.Fatalf("expected 3 HCAs, got %d", len(hcas))
|
||||||
|
}
|
||||||
|
// Sorted by name, so the order is predictable for the rest of the test.
|
||||||
|
if hcas[0].Name != "mlx5_0" || hcas[2].Name != "mlx5_2" {
|
||||||
|
t.Fatalf("HCAs are not sorted by name: %v %v", hcas[0].Name, hcas[2].Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
h := hcas[0]
|
||||||
|
if h.FirmwareVersion != "20.35.1012" {
|
||||||
|
t.Errorf("fw_ver=%q", h.FirmwareVersion)
|
||||||
|
}
|
||||||
|
if h.PCIAddr != "0000:0b:00.0" {
|
||||||
|
t.Errorf("PCIAddr=%q", h.PCIAddr)
|
||||||
|
}
|
||||||
|
if h.NUMANode != 1 {
|
||||||
|
t.Errorf("NUMANode=%d; GPU-to-HCA affinity depends on this", h.NUMANode)
|
||||||
|
}
|
||||||
|
|
||||||
|
p := h.Ports[0]
|
||||||
|
if p.State != "ACTIVE" || !p.Active() {
|
||||||
|
t.Errorf("state=%q", p.State)
|
||||||
|
}
|
||||||
|
if p.PhysState != "LinkUp" {
|
||||||
|
t.Errorf("phys_state=%q", p.PhysState)
|
||||||
|
}
|
||||||
|
if p.RateGbps != 200 {
|
||||||
|
t.Errorf("rate=%d", p.RateGbps)
|
||||||
|
}
|
||||||
|
if p.LinkLayer != LinkInfiniBand {
|
||||||
|
t.Errorf("link_layer=%q", p.LinkLayer)
|
||||||
|
}
|
||||||
|
if p.GIDPrefix != "fec0:0000:0000:000a" {
|
||||||
|
t.Errorf("GIDPrefix=%q; it must be lowercased and cut to four groups", p.GIDPrefix)
|
||||||
|
}
|
||||||
|
if p.LID != 26 {
|
||||||
|
t.Errorf("LID=%d, expected 0x1a to parse as 26", p.LID)
|
||||||
|
}
|
||||||
|
if p.SMLID != 1 {
|
||||||
|
t.Errorf("SMLID=%d, expected a decimal value to parse too", p.SMLID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestActivePortsExcludesDown(t *testing.T) {
|
||||||
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
h := hcas[1] // mlx5_1
|
||||||
|
if len(h.Ports) != 2 {
|
||||||
|
t.Fatalf("expected 2 ports, got %d", len(h.Ports))
|
||||||
|
}
|
||||||
|
active := h.ActivePorts()
|
||||||
|
if len(active) != 1 || active[0].Num != 1 {
|
||||||
|
t.Errorf("a DOWN port was reported as active: %+v", active)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRoCEIsDistinguished(t *testing.T) {
|
||||||
|
// RoCE is usable but behaves differently from native IB; the placement
|
||||||
|
// logic needs to be able to tell them apart.
|
||||||
|
hcas, err := DiscoverIB(fakeSysfs(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := hcas[2].Ports[0].LinkLayer; got != LinkEthernet {
|
||||||
|
t.Errorf("mlx5_2 link_layer=%q, expected Ethernet (RoCE)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMissingRootIsNotAnError(t *testing.T) {
|
||||||
|
// A node without InfiniBand is perfectly normal; it just forms its own
|
||||||
|
// single-node island.
|
||||||
|
hcas, err := DiscoverIB(filepath.Join(t.TempDir(), "nonexistent"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("a node without IB should not be an error: %v", err)
|
||||||
|
}
|
||||||
|
if len(hcas) != 0 {
|
||||||
|
t.Errorf("expected no HCAs, got %d", len(hcas))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnreadableAdapterDoesNotHideOthers(t *testing.T) {
|
||||||
|
root := fakeSysfs(t)
|
||||||
|
// An adapter with no ports directory at all.
|
||||||
|
if err := os.MkdirAll(filepath.Join(root, "mlx5_broken"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
hcas, err := DiscoverIB(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(hcas) != 3 {
|
||||||
|
t.Errorf("a broken adapter changed the result: got %d HCAs", len(hcas))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseHelpers(t *testing.T) {
|
||||||
|
t.Run("rate", func(t *testing.T) {
|
||||||
|
for in, want := range map[string]int{
|
||||||
|
"200 Gb/sec (4X HDR)": 200,
|
||||||
|
"100 Gb/sec (4X EDR)": 100,
|
||||||
|
"2.5 Gb/sec (1X SDR)": 2,
|
||||||
|
"": 0,
|
||||||
|
"garbage": 0,
|
||||||
|
} {
|
||||||
|
if got := parseRate(in); got != want {
|
||||||
|
t.Errorf("parseRate(%q)=%d, want %d", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("state", func(t *testing.T) {
|
||||||
|
for in, want := range map[string]string{
|
||||||
|
"4: ACTIVE": "ACTIVE",
|
||||||
|
"1: DOWN": "DOWN",
|
||||||
|
"5: LinkUp": "LinkUp",
|
||||||
|
"ACTIVE": "ACTIVE",
|
||||||
|
"": "",
|
||||||
|
} {
|
||||||
|
if got := parseEnumeratedState(in); got != want {
|
||||||
|
t.Errorf("parseEnumeratedState(%q)=%q, want %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("hex", func(t *testing.T) {
|
||||||
|
for in, want := range map[string]int{
|
||||||
|
"0x1a": 26, "0X1A": 26, "26": 26, "": 0, "zzz": 0,
|
||||||
|
} {
|
||||||
|
if got := parseMaybeHex(in); got != want {
|
||||||
|
t.Errorf("parseMaybeHex(%q)=%d, want %d", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("gid", func(t *testing.T) {
|
||||||
|
if got := parseGIDPrefix("FE80:0000:0000:0000:0002:C903:00A0:1234"); got != "fe80:0000:0000:0000" {
|
||||||
|
t.Errorf("parseGIDPrefix=%q", got)
|
||||||
|
}
|
||||||
|
if got := parseGIDPrefix("short:gid"); got != "" {
|
||||||
|
t.Errorf("a malformed GID should yield an empty prefix, got %q", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestJSONShape(t *testing.T) {
|
||||||
|
inv := Inventory{
|
||||||
|
Source: "nvidia-smi",
|
||||||
|
GPUs: []GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, ComputeCapability: "9.0"}},
|
||||||
|
}
|
||||||
|
b, err := json.Marshal(inv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var out struct {
|
||||||
|
Source string `json:"source"`
|
||||||
|
GPUs []struct {
|
||||||
|
Index int `json:"index"`
|
||||||
|
Class string `json:"class"`
|
||||||
|
Mem int `json:"memory_total_mb"`
|
||||||
|
} `json:"gpus"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(b, &out); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if out.Source != "nvidia-smi" || len(out.GPUs) != 1 {
|
||||||
|
t.Fatalf("shape wrong: %s", b)
|
||||||
|
}
|
||||||
|
// The derived class is what decides which GPUs are interchangeable, so a
|
||||||
|
// consumer must not have to recompute it.
|
||||||
|
if out.GPUs[0].Class != "h100-80g" {
|
||||||
|
t.Errorf("class=%q, should be included in the JSON", out.GPUs[0].Class)
|
||||||
|
}
|
||||||
|
if out.GPUs[0].Mem != 81559 {
|
||||||
|
t.Errorf("memory_total_mb=%d", out.GPUs[0].Mem)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MemoryTotalMB reports the machine's total RAM.
|
||||||
|
//
|
||||||
|
// It exists because placement had no idea how much a node could hold. A CPU
|
||||||
|
// profile has no card to be free, so every ready node looked like a candidate:
|
||||||
|
// a 14B model whose working set is 24 GB would have been placed just as
|
||||||
|
// happily on a node with 8 GB, which reports itself perfectly healthy right up
|
||||||
|
// until the OOM killer arrives. Labels were the stopgap; this is the fact they
|
||||||
|
// were standing in for.
|
||||||
|
//
|
||||||
|
// Read from /proc/meminfo, which is Linux-only — every node is. On anything
|
||||||
|
// else it returns 0, which callers treat as "unknown" and therefore no
|
||||||
|
// constraint, exactly as an older agent that does not report it at all.
|
||||||
|
func MemoryTotalMB() int { return memTotalMBFrom("/proc/meminfo") }
|
||||||
|
|
||||||
|
func memTotalMBFrom(path string) int {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
sc := bufio.NewScanner(f)
|
||||||
|
for sc.Scan() {
|
||||||
|
rest, ok := strings.CutPrefix(sc.Text(), "MemTotal:")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fields := strings.Fields(rest)
|
||||||
|
if len(fields) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
kb, err := strconv.Atoi(fields[0])
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
// MemTotal is what the kernel can hand out, already short of the
|
||||||
|
// installed total by the firmware's reservations. That is the right
|
||||||
|
// number for deciding what fits.
|
||||||
|
return kb / 1024
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMemTotalIsReadFromMeminfo(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
p := filepath.Join(dir, "meminfo")
|
||||||
|
// The real shape, including the fields before and after, so the parser is
|
||||||
|
// not tested against something tidier than it will see.
|
||||||
|
if err := os.WriteFile(p, []byte(
|
||||||
|
"MemTotal: 32876244 kB\nMemFree: 1234 kB\nMemAvailable: 5678 kB\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, want := memTotalMBFrom(p), 32876244/1024; got != want {
|
||||||
|
t.Errorf("memTotalMBFrom = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnreadableMeminfoReportsUnknownRatherThanZeroMemory(t *testing.T) {
|
||||||
|
// Zero means "did not say" everywhere downstream, and placement treats it
|
||||||
|
// as no constraint. A node with genuinely no memory is not a case.
|
||||||
|
if got := memTotalMBFrom(filepath.Join(t.TempDir(), "absent")); got != 0 {
|
||||||
|
t.Errorf("got %d for a missing file", got)
|
||||||
|
}
|
||||||
|
dir := t.TempDir()
|
||||||
|
p := filepath.Join(dir, "meminfo")
|
||||||
|
for _, junk := range []string{"", "MemTotal:\n", "MemTotal: not-a-number kB\n", "Other: 1 kB\n"} {
|
||||||
|
if err := os.WriteFile(p, []byte(junk), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := memTotalMBFrom(p); got != 0 {
|
||||||
|
t.Errorf("got %d for %q", got, junk)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Storage is one filesystem a node keeps weights on.
|
||||||
|
//
|
||||||
|
// One entry per filesystem rather than per configured directory, because free
|
||||||
|
// space belongs to the disk and not to the path. Two model directories on the
|
||||||
|
// same disk are one pool of bytes, and reporting them separately would let the
|
||||||
|
// control plane add 24 GB to 24 GB and place a 40 GB model on a node with 24.
|
||||||
|
type Storage struct {
|
||||||
|
// Path is one directory on this filesystem — the first one configured that
|
||||||
|
// landed here, so the operator recognises it.
|
||||||
|
Path string `json:"path"`
|
||||||
|
// FreeBytes is what a non-root process can still write. Deliberately not
|
||||||
|
// the root reserve: the agent does not run as root, so the reserve is space
|
||||||
|
// it cannot have.
|
||||||
|
FreeBytes int64 `json:"free_bytes"`
|
||||||
|
// TotalBytes is the filesystem's size, so a reading of "8 GB free" can be
|
||||||
|
// told apart on a 10 TB filer and on a 10 GB root partition.
|
||||||
|
TotalBytes int64 `json:"total_bytes"`
|
||||||
|
// Shared marks a filesystem the node did not have to fill. Free space on
|
||||||
|
// one is shared with every other node using it, so two nodes reporting
|
||||||
|
// 500 GB free on the same filer have 500 GB between them, not a terabyte.
|
||||||
|
Shared bool `json:"shared,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScanStorage reports the filesystems behind the given model directories.
|
||||||
|
//
|
||||||
|
// Empty in, empty out, and a directory that cannot be read is skipped rather
|
||||||
|
// than reported as full: a mount that has not come back yet looks exactly like
|
||||||
|
// a missing directory, and claiming zero free bytes would take the node out of
|
||||||
|
// placement for a reason that is not true. Absent is not zero here, as
|
||||||
|
// everywhere else in this system.
|
||||||
|
func ScanStorage(dirs []string) []Storage {
|
||||||
|
var out []Storage
|
||||||
|
seen := map[uint64]int{} // filesystem id -> index in out
|
||||||
|
|
||||||
|
for _, dir := range dirs {
|
||||||
|
if dir == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
dev, ok := filesystemID(dir)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if i, dup := seen[dev]; dup {
|
||||||
|
// Same disk, already counted. Keep the first path: it is the one
|
||||||
|
// the operator wrote first and so the one they will recognise.
|
||||||
|
_ = i
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
var st syscall.Statfs_t
|
||||||
|
if err := syscall.Statfs(dir, &st); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Bsize is int32 on some platforms and int64 on others, which is why
|
||||||
|
// every field here is widened explicitly rather than assumed.
|
||||||
|
block := int64(st.Bsize)
|
||||||
|
seen[dev] = len(out)
|
||||||
|
out = append(out, Storage{
|
||||||
|
Path: dir,
|
||||||
|
FreeBytes: int64(st.Bavail) * block, //nolint:gosec,unconvert // widened on purpose
|
||||||
|
TotalBytes: int64(st.Blocks) * block, //nolint:gosec,unconvert // widened on purpose
|
||||||
|
Shared: isNetworkFS(dir),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// filesystemID identifies the disk a path is on.
|
||||||
|
//
|
||||||
|
// By device number rather than by the statfs filesystem id, because the field
|
||||||
|
// holding that id is spelled differently on Linux and on macOS and this has to
|
||||||
|
// compile on both — the agent runs on one and is developed on the other.
|
||||||
|
func filesystemID(path string) (uint64, bool) {
|
||||||
|
fi, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
st, ok := fi.Sys().(*syscall.Stat_t)
|
||||||
|
if !ok {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
return uint64(st.Dev), true //nolint:gosec,unconvert // int32 on macOS, uint64 on Linux
|
||||||
|
}
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTwoDirectoriesOnOneDiskAreOnePoolOfBytes(t *testing.T) {
|
||||||
|
// The reason this matters: free space belongs to the disk, not to the path.
|
||||||
|
// Counting two directories separately lets the control plane add 24 GB to
|
||||||
|
// 24 GB and place a 40 GB model on a node that has 24.
|
||||||
|
base := t.TempDir()
|
||||||
|
a := filepath.Join(base, "models-a")
|
||||||
|
b := filepath.Join(base, "models-b")
|
||||||
|
for _, d := range []string{a, b} {
|
||||||
|
if err := os.Mkdir(d, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
got := ScanStorage([]string{a, b})
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("two directories on one filesystem reported as %d pools: %+v", len(got), got)
|
||||||
|
}
|
||||||
|
if got[0].Path != a {
|
||||||
|
t.Errorf("kept %q rather than the first directory configured", got[0].Path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStorageReportsSomethingPlausible(t *testing.T) {
|
||||||
|
got := ScanStorage([]string{t.TempDir()})
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("a real directory reported %d filesystems", len(got))
|
||||||
|
}
|
||||||
|
s := got[0]
|
||||||
|
if s.TotalBytes <= 0 {
|
||||||
|
t.Error("a filesystem with no size at all")
|
||||||
|
}
|
||||||
|
if s.FreeBytes < 0 || s.FreeBytes > s.TotalBytes {
|
||||||
|
t.Errorf("free %d of total %d", s.FreeBytes, s.TotalBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMissingDirectoryIsNotAFullDisk(t *testing.T) {
|
||||||
|
// A mount that has not come back yet looks exactly like a missing
|
||||||
|
// directory. Reporting it as zero free bytes would take the node out of
|
||||||
|
// placement for a reason that is not true; reporting nothing says only
|
||||||
|
// that nothing is known, which is the honest answer.
|
||||||
|
got := ScanStorage([]string{filepath.Join(t.TempDir(), "never-mounted")})
|
||||||
|
if len(got) != 0 {
|
||||||
|
t.Errorf("a missing directory was reported as a filesystem: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoDirectoriesConstrainNothing(t *testing.T) {
|
||||||
|
if got := ScanStorage(nil); len(got) != 0 {
|
||||||
|
t.Errorf("scanning nothing found %+v", got)
|
||||||
|
}
|
||||||
|
if got := ScanStorage([]string{""}); len(got) != 0 {
|
||||||
|
t.Errorf("an empty path found %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,137 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// LocalWeights is one model file or directory a node can see.
|
||||||
|
type LocalWeights struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
SizeBytes int64 `json:"size_bytes"`
|
||||||
|
// Shared marks weights on a filesystem this node did not have to fill.
|
||||||
|
//
|
||||||
|
// Worth distinguishing: twenty nodes each holding their own copy and twenty
|
||||||
|
// nodes seeing one filer are the same list of paths and completely
|
||||||
|
// different situations — one has twenty copies of a 141 GB model and the
|
||||||
|
// other has one.
|
||||||
|
Shared bool `json:"shared,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScanWeights lists the model files under the given directories.
|
||||||
|
//
|
||||||
|
// Storage is the part of this system that will differ at every installation —
|
||||||
|
// a Lustre mount, an image exported over the fabric, local NVMe, an rsync from
|
||||||
|
// a laptop — so llmgw does not try to own it. It observes the result instead:
|
||||||
|
// whatever put the file there, the question placement needs answered is only
|
||||||
|
// whether it is present.
|
||||||
|
//
|
||||||
|
// Scanning nothing is the default and constrains nothing. A node with no
|
||||||
|
// directories configured behaves exactly as every node did before this existed,
|
||||||
|
// which is what makes the whole feature optional.
|
||||||
|
func ScanWeights(dirs []string) []LocalWeights {
|
||||||
|
var out []LocalWeights
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
|
||||||
|
for _, dir := range dirs {
|
||||||
|
if dir == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
shared := isNetworkFS(dir)
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
if err != nil {
|
||||||
|
// A directory that is not there is not an error to report upward:
|
||||||
|
// a mount that has not come back yet looks exactly like this, and
|
||||||
|
// the node is simply one without those weights until it does.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
path := filepath.Join(dir, e.Name())
|
||||||
|
if _, dup := seen[path]; dup {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
size, ok := weightSize(path, e)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[path] = struct{}{}
|
||||||
|
out = append(out, LocalWeights{Path: path, SizeBytes: size, Shared: shared})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path })
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// weightSize measures a candidate and says whether it is one at all.
|
||||||
|
//
|
||||||
|
// A model is either a single file — a .gguf, usually — or a directory of
|
||||||
|
// shards and a config, which is what every HuggingFace checkout looks like.
|
||||||
|
// Anything else in the directory is somebody's notes and is not reported.
|
||||||
|
func weightSize(path string, e fs.DirEntry) (int64, bool) {
|
||||||
|
if e.IsDir() {
|
||||||
|
// A directory counts only if it actually holds weights. An empty one
|
||||||
|
// is a half-finished download, and reporting it as present would have
|
||||||
|
// placement choose a node whose engine then fails to load.
|
||||||
|
var total int64
|
||||||
|
var found bool
|
||||||
|
_ = filepath.WalkDir(path, func(p string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil || d.IsDir() {
|
||||||
|
return nil //nolint:nilerr // an unreadable subtree is not fatal
|
||||||
|
}
|
||||||
|
if isWeightFile(p) {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
if info, err := d.Info(); err == nil {
|
||||||
|
total += info.Size()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return total, found
|
||||||
|
}
|
||||||
|
if !isWeightFile(path) {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
info, err := e.Info()
|
||||||
|
if err != nil {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
return info.Size(), true
|
||||||
|
}
|
||||||
|
|
||||||
|
func isWeightFile(path string) bool {
|
||||||
|
switch strings.ToLower(filepath.Ext(path)) {
|
||||||
|
case ".gguf", ".safetensors", ".bin", ".pt", ".pth":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// isNetworkFS reports whether a path is on a filesystem the node did not fill.
|
||||||
|
//
|
||||||
|
// By magic number rather than by parsing mounts, because the mount table says
|
||||||
|
// what was asked for and this says what the kernel actually has. The list is
|
||||||
|
// the network filesystems that turn up in front of GPUs: NFS, Lustre, GPFS,
|
||||||
|
// CephFS, GlusterFS and SMB. Anything unrecognised is treated as local, which
|
||||||
|
// is the conservative direction — a shared filer mistaken for local storage
|
||||||
|
// costs an unnecessary copy, while local storage mistaken for shared would have
|
||||||
|
// the cluster believe a model is everywhere when it is on one node.
|
||||||
|
func isNetworkFS(path string) bool {
|
||||||
|
var st syscall.Statfs_t
|
||||||
|
if err := syscall.Statfs(path, &st); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch uint32(st.Type) { //nolint:unconvert // Type is int64 on some platforms
|
||||||
|
case 0x6969, // NFS
|
||||||
|
0x0BD00BD0, // Lustre
|
||||||
|
0x47504653, // GPFS
|
||||||
|
0x00C36400, // CephFS
|
||||||
|
0x65735546, // FUSE, which is how GlusterFS and many object mounts appear
|
||||||
|
0xFF534D42: // SMB/CIFS
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func write(t *testing.T, path string, size int) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, make([]byte, size), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestScanFindsFilesAndDirectoriesOfWeights(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
write(t, filepath.Join(dir, "qwen-3b-q4.gguf"), 1024)
|
||||||
|
// A HuggingFace checkout: a directory of shards rather than one file.
|
||||||
|
write(t, filepath.Join(dir, "Llama-3.3-70B", "model-00001.safetensors"), 2048)
|
||||||
|
write(t, filepath.Join(dir, "Llama-3.3-70B", "config.json"), 64)
|
||||||
|
|
||||||
|
got := ScanWeights([]string{dir})
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("found %d entries: %+v", len(got), got)
|
||||||
|
}
|
||||||
|
byName := map[string]int64{}
|
||||||
|
for _, w := range got {
|
||||||
|
byName[filepath.Base(w.Path)] = w.SizeBytes
|
||||||
|
}
|
||||||
|
if byName["qwen-3b-q4.gguf"] != 1024 {
|
||||||
|
t.Errorf("gguf size = %d", byName["qwen-3b-q4.gguf"])
|
||||||
|
}
|
||||||
|
// The directory's size is everything in it, config included: what matters
|
||||||
|
// is how much would have to be moved to put it somewhere else.
|
||||||
|
if byName["Llama-3.3-70B"] != 2048+64 {
|
||||||
|
t.Errorf("checkout size = %d, want %d", byName["Llama-3.3-70B"], 2048+64)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnEmptyDirectoryIsNotWeights(t *testing.T) {
|
||||||
|
// A half-finished download looks exactly like this, and reporting it as
|
||||||
|
// present would have placement choose a node whose engine then fails to
|
||||||
|
// load — the failure this whole feature exists to avoid.
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.MkdirAll(filepath.Join(dir, "downloading"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
write(t, filepath.Join(dir, "downloading", "README.md"), 10)
|
||||||
|
|
||||||
|
if got := ScanWeights([]string{dir}); len(got) != 0 {
|
||||||
|
t.Errorf("a directory with no weight files was reported: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNotesAndScriptsAreNotReportedAsModels(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
write(t, filepath.Join(dir, "notes.txt"), 10)
|
||||||
|
write(t, filepath.Join(dir, "fetch.sh"), 10)
|
||||||
|
write(t, filepath.Join(dir, "real.gguf"), 10)
|
||||||
|
|
||||||
|
got := ScanWeights([]string{dir})
|
||||||
|
if len(got) != 1 || filepath.Base(got[0].Path) != "real.gguf" {
|
||||||
|
t.Errorf("scan returned %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMissingDirectoryIsNotAnError(t *testing.T) {
|
||||||
|
// A mount that has not come back yet looks exactly like a directory that is
|
||||||
|
// not there, and the node is simply one without those weights until it is.
|
||||||
|
got := ScanWeights([]string{filepath.Join(t.TempDir(), "never-existed")})
|
||||||
|
if len(got) != 0 {
|
||||||
|
t.Errorf("got %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestScanningNothingIsTheDefaultAndConstrainsNothing(t *testing.T) {
|
||||||
|
// What makes the feature optional: an agent with no directories configured
|
||||||
|
// behaves exactly as every agent did before this existed.
|
||||||
|
if got := ScanWeights(nil); len(got) != 0 {
|
||||||
|
t.Errorf("got %+v", got)
|
||||||
|
}
|
||||||
|
if got := ScanWeights([]string{""}); len(got) != 0 {
|
||||||
|
t.Errorf("an empty directory name produced %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheSamePathIsReportedOnce(t *testing.T) {
|
||||||
|
// Two configured directories can overlap, or be the same mount reached two
|
||||||
|
// ways. Counting a model twice would overstate what a fabric holds.
|
||||||
|
dir := t.TempDir()
|
||||||
|
write(t, filepath.Join(dir, "m.gguf"), 10)
|
||||||
|
if got := ScanWeights([]string{dir, dir}); len(got) != 1 {
|
||||||
|
t.Errorf("the same directory twice produced %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLocalStorageIsNeverMistakenForShared(t *testing.T) {
|
||||||
|
// The conservative direction. A shared filer read as local costs an
|
||||||
|
// unnecessary copy; local read as shared would have the cluster believe a
|
||||||
|
// model is everywhere when it is on one node.
|
||||||
|
dir := t.TempDir()
|
||||||
|
write(t, filepath.Join(dir, "m.gguf"), 10)
|
||||||
|
got := ScanWeights([]string{dir})
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("got %+v", got)
|
||||||
|
}
|
||||||
|
if got[0].Shared {
|
||||||
|
t.Error("a temporary directory on local disk was reported as shared")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,323 @@
|
|||||||
|
// Package engines builds the command lines and environments that start
|
||||||
|
// inference engines (plan §5.3).
|
||||||
|
//
|
||||||
|
// Nothing here starts a process; that is the supervisor's job. Keeping command
|
||||||
|
// construction pure makes it testable without GPUs, which matters because the
|
||||||
|
// environment generation is where multi-node runs go wrong: a single missing
|
||||||
|
// NCCL variable turns a working tensor-parallel group into one that hangs on
|
||||||
|
// the first all-reduce.
|
||||||
|
package engines
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Profile is how a model runs on a GPU class. It mirrors the catalog profile
|
||||||
|
// (plan §6.1), reduced to what launching needs.
|
||||||
|
type Profile struct {
|
||||||
|
Name string
|
||||||
|
Engine string
|
||||||
|
GPUClass string
|
||||||
|
|
||||||
|
// TP and PP are tensor and pipeline parallel degrees.
|
||||||
|
TP int
|
||||||
|
PP int
|
||||||
|
// GPUsPerNode is how many GPUs each shard occupies.
|
||||||
|
GPUsPerNode int
|
||||||
|
// Nodes is how many nodes the replica spans. Above 1, a fabric is required.
|
||||||
|
Nodes int
|
||||||
|
|
||||||
|
// Args are passed through to the engine untouched.
|
||||||
|
Args []string
|
||||||
|
// Env overrides anything the engine adapter generates.
|
||||||
|
Env map[string]string
|
||||||
|
|
||||||
|
ModelPath string
|
||||||
|
// ServedModelName is the name clients use, which need not match the path.
|
||||||
|
ServedModelName string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShardSpec is one shard of one replica on this node.
|
||||||
|
type ShardSpec struct {
|
||||||
|
ReplicaID string
|
||||||
|
ShardID string
|
||||||
|
// Rank is the node rank; 0 is the leader and serves HTTP.
|
||||||
|
Rank int
|
||||||
|
NNodes int
|
||||||
|
|
||||||
|
// LeaderAddr and LeaderPort are the distributed init endpoint, always on the
|
||||||
|
// management network rather than the fabric: NCCL bootstraps over TCP and
|
||||||
|
// only then moves to RDMA.
|
||||||
|
LeaderAddr string
|
||||||
|
LeaderPort int
|
||||||
|
|
||||||
|
// GPUIndices are the local GPUs this shard uses.
|
||||||
|
GPUIndices []int
|
||||||
|
// HCAs are the adapters of the chosen fabric, and only that fabric. A shard
|
||||||
|
// must never be given HCAs from two fabrics.
|
||||||
|
HCAs []string
|
||||||
|
// MgmtInterface is the ethernet interface used for bootstrap traffic.
|
||||||
|
MgmtInterface string
|
||||||
|
|
||||||
|
HTTPPort int
|
||||||
|
Profile Profile
|
||||||
|
|
||||||
|
// Pool and Fabric are control plane concepts the agent cannot derive. They
|
||||||
|
// are carried so the agent can report them back after a restart, and are
|
||||||
|
// not used when building a command.
|
||||||
|
Pool string
|
||||||
|
Fabric string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Leader reports whether this shard serves HTTP for the replica.
|
||||||
|
func (s ShardSpec) Leader() bool { return s.Rank == 0 }
|
||||||
|
|
||||||
|
// Command is a fully resolved launch: what to exec, with which environment.
|
||||||
|
type Command struct {
|
||||||
|
Path string
|
||||||
|
Args []string
|
||||||
|
// Env is the complete environment, not a delta.
|
||||||
|
Env []string
|
||||||
|
// Name distinguishes the processes of a multi-process shard in logs.
|
||||||
|
Name string
|
||||||
|
// WaitForLeaderPort makes the supervisor hold the next command until
|
||||||
|
// LeaderAddr:LeaderPort accepts a connection.
|
||||||
|
//
|
||||||
|
// It exists for Ray: the head has to be listening before a worker can join,
|
||||||
|
// and both sides express the same wait because both dial the same address.
|
||||||
|
WaitForLeaderPort bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Engine builds launch commands for one inference engine.
|
||||||
|
type Engine interface {
|
||||||
|
// Name is the engine's identifier in the catalog.
|
||||||
|
Name() string
|
||||||
|
// Validate rejects a profile that cannot work on these GPUs, before
|
||||||
|
// anything is started.
|
||||||
|
Validate(p Profile, gpus []GPUInfo) error
|
||||||
|
|
||||||
|
// Commands returns the processes that make up one shard, in start order.
|
||||||
|
//
|
||||||
|
// Most shards are a single process. vLLM under Ray is not: the leader runs
|
||||||
|
// a Ray head and then the server against it, and a worker runs only the Ray
|
||||||
|
// join. Returning a sequence rather than one command is what lets the
|
||||||
|
// supervisor treat both the same way — every command but the last is a
|
||||||
|
// sidecar kept alive for the shard's lifetime, and the last one's exit is
|
||||||
|
// the shard's exit.
|
||||||
|
Commands(s ShardSpec) ([]Command, error)
|
||||||
|
|
||||||
|
// ServesHTTP reports whether this rank answers the OpenAI API.
|
||||||
|
//
|
||||||
|
// In a multi-node replica only the leader does. Health-checking a worker's
|
||||||
|
// port would time out and fail a replica that is coming up perfectly well,
|
||||||
|
// which is exactly what happened before this existed.
|
||||||
|
ServesHTTP(s ShardSpec) bool
|
||||||
|
|
||||||
|
// HealthPath and MetricsPath are appended to the shard's base URL.
|
||||||
|
HealthPath() string
|
||||||
|
MetricsPath() string
|
||||||
|
}
|
||||||
|
|
||||||
|
// GPUInfo is the subset of GPU facts validation needs.
|
||||||
|
type GPUInfo struct {
|
||||||
|
Index int
|
||||||
|
Name string
|
||||||
|
ComputeCapability string
|
||||||
|
MemoryTotalMB int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Registry holds the known engines.
|
||||||
|
type Registry struct {
|
||||||
|
engines map[string]Engine
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRegistry returns a registry with the built-in engines.
|
||||||
|
func NewRegistry(cfg Config) *Registry {
|
||||||
|
r := &Registry{engines: map[string]Engine{}}
|
||||||
|
r.Add(&VLLM{cfg: cfg.VLLM})
|
||||||
|
r.Add(&SGLang{cfg: cfg.SGLang})
|
||||||
|
r.Add(&LlamaCpp{cfg: cfg.LlamaCpp})
|
||||||
|
r.Add(&Sim{cfg: cfg.Sim})
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add registers an engine, replacing any engine of the same name.
|
||||||
|
func (r *Registry) Add(e Engine) { r.engines[e.Name()] = e }
|
||||||
|
|
||||||
|
// Get returns an engine by name.
|
||||||
|
func (r *Registry) Get(name string) (Engine, error) {
|
||||||
|
e, ok := r.engines[name]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("unknown engine %q (have %s)", name, strings.Join(r.Names(), ", "))
|
||||||
|
}
|
||||||
|
return e, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Names lists the registered engines.
|
||||||
|
func (r *Registry) Names() []string {
|
||||||
|
out := make([]string, 0, len(r.engines))
|
||||||
|
for n := range r.engines {
|
||||||
|
out = append(out, n)
|
||||||
|
}
|
||||||
|
slices.Sort(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config configures the engine adapters.
|
||||||
|
type Config struct {
|
||||||
|
VLLM VLLMConfig `koanf:"vllm"`
|
||||||
|
SGLang SGLangConfig `koanf:"sglang"`
|
||||||
|
LlamaCpp LlamaCppConfig `koanf:"llamacpp"`
|
||||||
|
Sim SimConfig `koanf:"sim"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- shared helpers ---
|
||||||
|
|
||||||
|
// baseEnv builds the environment every engine shares.
|
||||||
|
//
|
||||||
|
// This is the part of §5.3 the plan marks critical. The rules encoded here:
|
||||||
|
//
|
||||||
|
// - CUDA_VISIBLE_DEVICES pins the shard to its GPUs, so two shards on one
|
||||||
|
// node cannot fight over the same card.
|
||||||
|
// - NCCL_IB_HCA lists only the chosen fabric's adapters. Leaving it unset
|
||||||
|
// lets NCCL pick adapters from another fabric, where the peer is
|
||||||
|
// unreachable, and the job hangs instead of failing.
|
||||||
|
// - NCCL_SOCKET_IFNAME pins bootstrap to the management interface. NCCL
|
||||||
|
// otherwise probes every interface, including IB-over-IP addresses that
|
||||||
|
// cannot route between fabrics.
|
||||||
|
// - NCCL_IB_DISABLE is set when there is no fabric, so NCCL stops looking for
|
||||||
|
// one instead of timing out.
|
||||||
|
func baseEnv(s ShardSpec) map[string]string {
|
||||||
|
env := map[string]string{
|
||||||
|
"CUDA_VISIBLE_DEVICES": joinInts(s.GPUIndices),
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(s.HCAs) > 0 {
|
||||||
|
env["NCCL_IB_HCA"] = strings.Join(s.HCAs, ",")
|
||||||
|
env["NCCL_IB_DISABLE"] = "0"
|
||||||
|
} else {
|
||||||
|
// Single node, or a node with no fabric. Either way IB is not the path.
|
||||||
|
env["NCCL_IB_DISABLE"] = "1"
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.MgmtInterface != "" {
|
||||||
|
env["NCCL_SOCKET_IFNAME"] = s.MgmtInterface
|
||||||
|
env["GLOO_SOCKET_IFNAME"] = s.MgmtInterface
|
||||||
|
}
|
||||||
|
|
||||||
|
// Profile overrides win: they are the escape hatch for the per-cluster
|
||||||
|
// tuning no adapter can guess.
|
||||||
|
for k, v := range s.Profile.Env {
|
||||||
|
env[k] = v
|
||||||
|
}
|
||||||
|
return env
|
||||||
|
}
|
||||||
|
|
||||||
|
// materializeEnv merges generated variables over the current process
|
||||||
|
// environment and returns it in exec form.
|
||||||
|
//
|
||||||
|
// Inheriting matters: engines need PATH, HOME, HF_HOME, LD_LIBRARY_PATH and
|
||||||
|
// whatever else the operator set up in the unit file.
|
||||||
|
func materializeEnv(generated map[string]string) []string {
|
||||||
|
merged := map[string]string{}
|
||||||
|
for _, kv := range os.Environ() {
|
||||||
|
if k, v, ok := strings.Cut(kv, "="); ok {
|
||||||
|
merged[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for k, v := range generated {
|
||||||
|
merged[k] = v
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]string, 0, len(merged))
|
||||||
|
for k, v := range merged {
|
||||||
|
out = append(out, k+"="+v)
|
||||||
|
}
|
||||||
|
// Sorted so a command is reproducible and diffable in logs.
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func joinInts(v []int) string {
|
||||||
|
parts := make([]string, len(v))
|
||||||
|
for i, n := range v {
|
||||||
|
parts[i] = strconv.Itoa(n)
|
||||||
|
}
|
||||||
|
return strings.Join(parts, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateCommon holds the checks that apply to every engine.
|
||||||
|
func validateCommon(p Profile, gpus []GPUInfo) error {
|
||||||
|
if p.ModelPath == "" {
|
||||||
|
return fmt.Errorf("profile %q: model path is empty", p.Name)
|
||||||
|
}
|
||||||
|
if p.TP <= 0 {
|
||||||
|
return fmt.Errorf("profile %q: tensor parallel degree must be positive", p.Name)
|
||||||
|
}
|
||||||
|
if p.Nodes > 1 && p.GPUsPerNode <= 0 {
|
||||||
|
return fmt.Errorf("profile %q: gpus_per_node is required for a %d-node profile", p.Name, p.Nodes)
|
||||||
|
}
|
||||||
|
|
||||||
|
perNode := p.GPUsPerNode
|
||||||
|
if perNode == 0 {
|
||||||
|
perNode = p.TP
|
||||||
|
}
|
||||||
|
nodes := max(p.Nodes, 1)
|
||||||
|
if want := perNode * nodes; want != p.TP*max(p.PP, 1) {
|
||||||
|
return fmt.Errorf("profile %q: %d nodes x %d GPUs does not match TP %d x PP %d",
|
||||||
|
p.Name, nodes, perNode, p.TP, max(p.PP, 1))
|
||||||
|
}
|
||||||
|
if len(gpus) > 0 && len(gpus) < perNode {
|
||||||
|
return fmt.Errorf("profile %q: needs %d GPUs per node, node has %d", p.Name, perNode, len(gpus))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A tensor-parallel group must be uniform: mixing classes makes the slowest
|
||||||
|
// card set the pace for every collective, and mismatched memory makes the
|
||||||
|
// smallest card decide what fits.
|
||||||
|
if len(gpus) > 1 {
|
||||||
|
first := gpus[0].Name
|
||||||
|
for _, g := range gpus[1:] {
|
||||||
|
if g.Name != first {
|
||||||
|
return fmt.Errorf("profile %q: GPUs are not uniform (%q and %q); "+
|
||||||
|
"a tensor-parallel group may not mix GPU types", p.Name, first, g.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// requireComputeCapability rejects profiles whose quantization the hardware
|
||||||
|
// cannot execute.
|
||||||
|
func requireComputeCapability(p Profile, gpus []GPUInfo, minCap float64, why string) error {
|
||||||
|
for _, g := range gpus {
|
||||||
|
cap, err := strconv.ParseFloat(g.ComputeCapability, 64)
|
||||||
|
if err != nil {
|
||||||
|
// Unknown capability: let it through rather than block a launch on
|
||||||
|
// a parsing problem. The engine will refuse it clearly enough.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if cap < minCap {
|
||||||
|
return fmt.Errorf("profile %q: %s needs compute capability %.1f, GPU %d (%s) has %.1f",
|
||||||
|
p.Name, why, minCap, g.Index, g.Name, cap)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// quantizationOf extracts the --quantization argument, if present.
|
||||||
|
func quantizationOf(args []string) string {
|
||||||
|
for i, a := range args {
|
||||||
|
if a == "--quantization" && i+1 < len(args) {
|
||||||
|
return args[i+1]
|
||||||
|
}
|
||||||
|
if v, ok := strings.CutPrefix(a, "--quantization="); ok {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,425 @@
|
|||||||
|
package engines
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// buildOne returns the single command a shard produces, failing if it produced
|
||||||
|
// a different number.
|
||||||
|
func buildOne(t *testing.T, e Engine, s ShardSpec) Command {
|
||||||
|
t.Helper()
|
||||||
|
cmds := build(t, e, s)
|
||||||
|
if len(cmds) != 1 {
|
||||||
|
t.Fatalf("expected 1 command, got %d: %s", len(cmds), summarize(cmds))
|
||||||
|
}
|
||||||
|
return cmds[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildEngine returns the last command, which is the engine itself. The ones
|
||||||
|
// before it are sidecars.
|
||||||
|
func buildEngine(t *testing.T, e Engine, s ShardSpec) Command {
|
||||||
|
t.Helper()
|
||||||
|
cmds := build(t, e, s)
|
||||||
|
return cmds[len(cmds)-1]
|
||||||
|
}
|
||||||
|
|
||||||
|
func build(t *testing.T, e Engine, s ShardSpec) []Command {
|
||||||
|
t.Helper()
|
||||||
|
cmds, err := e.Commands(s)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(cmds) == 0 {
|
||||||
|
t.Fatal("no commands produced")
|
||||||
|
}
|
||||||
|
return cmds
|
||||||
|
}
|
||||||
|
|
||||||
|
// summarize keeps failure output readable: a Command carries the whole
|
||||||
|
// environment, and printing it buries the actual problem.
|
||||||
|
func summarize(cmds []Command) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for i, c := range cmds {
|
||||||
|
if i > 0 {
|
||||||
|
b.WriteString(" | ")
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, "%s: %s %s", c.Name, c.Path, strings.Join(c.Args, " "))
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// envMap turns exec-form environment back into a map for assertions.
|
||||||
|
func envMap(env []string) map[string]string {
|
||||||
|
m := map[string]string{}
|
||||||
|
for _, kv := range env {
|
||||||
|
if k, v, ok := strings.Cut(kv, "="); ok {
|
||||||
|
m[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// argValue returns the value following flag, or "" if the flag is absent.
|
||||||
|
func argValue(args []string, flag string) string {
|
||||||
|
i := slices.Index(args, flag)
|
||||||
|
if i < 0 || i+1 >= len(args) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return args[i+1]
|
||||||
|
}
|
||||||
|
|
||||||
|
func singleNodeSpec() ShardSpec {
|
||||||
|
return ShardSpec{
|
||||||
|
ReplicaID: "r1",
|
||||||
|
ShardID: "r1-0",
|
||||||
|
Rank: 0,
|
||||||
|
NNodes: 1,
|
||||||
|
GPUIndices: []int{0, 1, 2, 3},
|
||||||
|
MgmtInterface: "eth0",
|
||||||
|
HTTPPort: 20000,
|
||||||
|
Profile: Profile{
|
||||||
|
Name: "h100-tp4",
|
||||||
|
Engine: "vllm",
|
||||||
|
TP: 4,
|
||||||
|
ModelPath: "/models/llama",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func multiNodeSpec(rank int) ShardSpec {
|
||||||
|
s := singleNodeSpec()
|
||||||
|
s.Rank = rank
|
||||||
|
s.NNodes = 2
|
||||||
|
s.GPUIndices = []int{0, 1, 2, 3, 4, 5, 6, 7}
|
||||||
|
s.HCAs = []string{"mlx5_0", "mlx5_1"}
|
||||||
|
s.LeaderAddr = "10.0.0.10"
|
||||||
|
s.LeaderPort = 20001
|
||||||
|
s.Profile.TP = 16
|
||||||
|
s.Profile.GPUsPerNode = 8
|
||||||
|
s.Profile.Nodes = 2
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- environment generation: the part that breaks multi-node runs ---
|
||||||
|
|
||||||
|
func TestSingleNodeDisablesIB(t *testing.T) {
|
||||||
|
cmd := buildOne(t, &VLLM{}, singleNodeSpec())
|
||||||
|
env := envMap(cmd.Env)
|
||||||
|
|
||||||
|
if env["NCCL_IB_DISABLE"] != "1" {
|
||||||
|
t.Error("NCCL_IB_DISABLE must be 1 without a fabric, or NCCL waits for one that is not there")
|
||||||
|
}
|
||||||
|
if _, set := env["NCCL_IB_HCA"]; set {
|
||||||
|
t.Error("NCCL_IB_HCA was set without any HCAs")
|
||||||
|
}
|
||||||
|
if env["CUDA_VISIBLE_DEVICES"] != "0,1,2,3" {
|
||||||
|
t.Errorf("CUDA_VISIBLE_DEVICES=%q", env["CUDA_VISIBLE_DEVICES"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultiNodePinsFabric(t *testing.T) {
|
||||||
|
cmd := buildEngine(t, &VLLM{}, multiNodeSpec(0))
|
||||||
|
env := envMap(cmd.Env)
|
||||||
|
|
||||||
|
// Only the chosen fabric's adapters. Leaving this open lets NCCL pick an
|
||||||
|
// adapter on another fabric, where the peer is unreachable, and the job
|
||||||
|
// hangs rather than failing.
|
||||||
|
if env["NCCL_IB_HCA"] != "mlx5_0,mlx5_1" {
|
||||||
|
t.Errorf("NCCL_IB_HCA=%q", env["NCCL_IB_HCA"])
|
||||||
|
}
|
||||||
|
if env["NCCL_IB_DISABLE"] != "0" {
|
||||||
|
t.Errorf("NCCL_IB_DISABLE=%q with a fabric present", env["NCCL_IB_DISABLE"])
|
||||||
|
}
|
||||||
|
// Bootstrap must stay on the management network; NCCL otherwise probes
|
||||||
|
// every interface, including IB-over-IP that cannot route between fabrics.
|
||||||
|
if env["NCCL_SOCKET_IFNAME"] != "eth0" || env["GLOO_SOCKET_IFNAME"] != "eth0" {
|
||||||
|
t.Errorf("socket interfaces not pinned: %q / %q",
|
||||||
|
env["NCCL_SOCKET_IFNAME"], env["GLOO_SOCKET_IFNAME"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileEnvOverridesGenerated(t *testing.T) {
|
||||||
|
s := multiNodeSpec(0)
|
||||||
|
s.Profile.Env = map[string]string{
|
||||||
|
"NCCL_IB_HCA": "mlx5_9",
|
||||||
|
"NCCL_NET_GDR_LEVEL": "SYS",
|
||||||
|
}
|
||||||
|
cmd := buildEngine(t, &VLLM{}, s)
|
||||||
|
env := envMap(cmd.Env)
|
||||||
|
|
||||||
|
if env["NCCL_IB_HCA"] != "mlx5_9" {
|
||||||
|
t.Error("a profile override did not win; it is the escape hatch for per-cluster tuning")
|
||||||
|
}
|
||||||
|
if env["NCCL_NET_GDR_LEVEL"] != "SYS" {
|
||||||
|
t.Error("an extra profile variable was dropped")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessEnvironmentIsInherited(t *testing.T) {
|
||||||
|
// Engines need PATH, HF_HOME, LD_LIBRARY_PATH and whatever else the unit
|
||||||
|
// file set up.
|
||||||
|
t.Setenv("LLMGW_TEST_MARKER", "present")
|
||||||
|
cmd := buildOne(t, &VLLM{}, singleNodeSpec())
|
||||||
|
if envMap(cmd.Env)["LLMGW_TEST_MARKER"] != "present" {
|
||||||
|
t.Error("the process environment was not inherited")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnvIsSorted(t *testing.T) {
|
||||||
|
cmd := buildOne(t, &VLLM{}, singleNodeSpec())
|
||||||
|
if !slices.IsSorted(cmd.Env) {
|
||||||
|
t.Error("environment is not sorted; launches would not be reproducible in logs")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- vLLM ---
|
||||||
|
|
||||||
|
func TestVLLMSingleNodeArgs(t *testing.T) {
|
||||||
|
cmd := buildOne(t, &VLLM{cfg: VLLMConfig{Bin: "/opt/vllm/bin/vllm"}}, singleNodeSpec())
|
||||||
|
if cmd.Path != "/opt/vllm/bin/vllm" {
|
||||||
|
t.Errorf("Path=%q", cmd.Path)
|
||||||
|
}
|
||||||
|
if cmd.Args[0] != "serve" || cmd.Args[1] != "/models/llama" {
|
||||||
|
t.Errorf("args start wrong: %v", cmd.Args[:2])
|
||||||
|
}
|
||||||
|
if argValue(cmd.Args, "--tensor-parallel-size") != "4" {
|
||||||
|
t.Errorf("TP not passed: %v", cmd.Args)
|
||||||
|
}
|
||||||
|
if argValue(cmd.Args, "--port") != "20000" {
|
||||||
|
t.Errorf("port not passed: %v", cmd.Args)
|
||||||
|
}
|
||||||
|
// Single node means no distributed backend flag at all.
|
||||||
|
if slices.Contains(cmd.Args, "--distributed-executor-backend") {
|
||||||
|
t.Error("a distributed backend was selected for a single-node profile")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVLLMRayLeaderRunsTwoProcesses(t *testing.T) {
|
||||||
|
// The leader starts a Ray head and then the server against it. A single
|
||||||
|
// command would mean running vLLM against a cluster that does not exist.
|
||||||
|
v := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
|
||||||
|
cmds, err := v.Commands(multiNodeSpec(0))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(cmds) != 2 {
|
||||||
|
t.Fatalf("expected 2 commands for the Ray leader, got %d", len(cmds))
|
||||||
|
}
|
||||||
|
if cmds[0].Name != "ray" || !slices.Contains(cmds[0].Args, "--head") {
|
||||||
|
t.Errorf("first command is not the Ray head: %+v", cmds[0])
|
||||||
|
}
|
||||||
|
if !cmds[0].WaitForLeaderPort {
|
||||||
|
t.Error("the server would start before the head is listening")
|
||||||
|
}
|
||||||
|
if cmds[1].Name != "vllm" || argValue(cmds[1].Args, "--distributed-executor-backend") != "ray" {
|
||||||
|
t.Errorf("second command is not the vLLM server: %+v", cmds[1])
|
||||||
|
}
|
||||||
|
if got := envMap(cmds[1].Env)["RAY_ADDRESS"]; got != "10.0.0.10:20001" {
|
||||||
|
t.Errorf("RAY_ADDRESS=%q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVLLMRayWorkerRunsOnlyRay(t *testing.T) {
|
||||||
|
// A Ray worker runs no vLLM process: the cluster hands its GPUs to the
|
||||||
|
// leader. Starting a second server there would bind a port nobody uses and
|
||||||
|
// compete for the same cards.
|
||||||
|
v := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
|
||||||
|
cmds, err := v.Commands(multiNodeSpec(1))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(cmds) != 1 {
|
||||||
|
t.Fatalf("expected 1 command for a Ray worker, got %d: %+v", len(cmds), cmds)
|
||||||
|
}
|
||||||
|
if cmds[0].Name != "ray" || slices.Contains(cmds[0].Args, "--head") {
|
||||||
|
t.Errorf("worker command wrong: %+v", cmds[0])
|
||||||
|
}
|
||||||
|
if argValue(cmds[0].Args, "--address") != "10.0.0.10:20001" {
|
||||||
|
t.Errorf("worker is not joining the leader: %v", cmds[0].Args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVLLMMPMode(t *testing.T) {
|
||||||
|
v := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}
|
||||||
|
cmd := buildOne(t, v, multiNodeSpec(0))
|
||||||
|
if argValue(cmd.Args, "--distributed-executor-backend") != "mp" {
|
||||||
|
t.Errorf("backend not set: %v", cmd.Args)
|
||||||
|
}
|
||||||
|
env := envMap(cmd.Env)
|
||||||
|
if env["VLLM_HOST_IP"] != "10.0.0.10" || env["VLLM_PORT"] != "20001" {
|
||||||
|
t.Errorf("dist init not configured: %q %q", env["VLLM_HOST_IP"], env["VLLM_PORT"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVLLMUnknownLaunchMode(t *testing.T) {
|
||||||
|
v := &VLLM{cfg: VLLMConfig{LaunchMode: "sorcery"}}
|
||||||
|
if _, err := v.Commands(multiNodeSpec(0)); err == nil {
|
||||||
|
t.Error("an unknown launch mode was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOnlyLeaderServesHTTP(t *testing.T) {
|
||||||
|
// The fix for the bug that stopped multi-node replicas coming up at all:
|
||||||
|
// health-checking a worker's port times out and fails a healthy replica.
|
||||||
|
for _, e := range []Engine{&VLLM{}, &SGLang{}} {
|
||||||
|
if !e.ServesHTTP(multiNodeSpec(0)) {
|
||||||
|
t.Errorf("%s: the leader must serve HTTP", e.Name())
|
||||||
|
}
|
||||||
|
if e.ServesHTTP(multiNodeSpec(1)) {
|
||||||
|
t.Errorf("%s: a worker must not be expected to serve HTTP", e.Name())
|
||||||
|
}
|
||||||
|
if !e.ServesHTTP(singleNodeSpec()) {
|
||||||
|
t.Errorf("%s: a single-node shard must serve HTTP", e.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoRayForSingleNodeOrMPMode(t *testing.T) {
|
||||||
|
ray := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
|
||||||
|
if cmds := buildOne(t, ray, singleNodeSpec()); cmds.Name != "vllm" {
|
||||||
|
t.Errorf("a single-node profile was given a Ray process: %+v", cmds)
|
||||||
|
}
|
||||||
|
mp := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}
|
||||||
|
if cmds := buildOne(t, mp, multiNodeSpec(0)); cmds.Name != "vllm" {
|
||||||
|
t.Errorf("mp mode was given a Ray process: %+v", cmds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- SGLang ---
|
||||||
|
|
||||||
|
func TestSGLangMultiNodeArgs(t *testing.T) {
|
||||||
|
cmd := buildOne(t, &SGLang{cfg: SGLangConfig{Python: "/opt/sglang/bin/python"}}, multiNodeSpec(1))
|
||||||
|
if cmd.Path != "/opt/sglang/bin/python" {
|
||||||
|
t.Errorf("Path=%q", cmd.Path)
|
||||||
|
}
|
||||||
|
if cmd.Args[0] != "-m" || cmd.Args[1] != "sglang.launch_server" {
|
||||||
|
t.Errorf("not launched as a module: %v", cmd.Args[:2])
|
||||||
|
}
|
||||||
|
if argValue(cmd.Args, "--nnodes") != "2" || argValue(cmd.Args, "--node-rank") != "1" {
|
||||||
|
t.Errorf("multi-node flags wrong: %v", cmd.Args)
|
||||||
|
}
|
||||||
|
if argValue(cmd.Args, "--dist-init-addr") != "10.0.0.10:20001" {
|
||||||
|
t.Errorf("dist-init-addr wrong: %v", cmd.Args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) {
|
||||||
|
cmd := buildOne(t, &SGLang{}, singleNodeSpec())
|
||||||
|
for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {
|
||||||
|
if slices.Contains(cmd.Args, flag) {
|
||||||
|
t.Errorf("%s was passed for a single-node profile", flag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileArgsComeLast(t *testing.T) {
|
||||||
|
// Operators must be able to override anything the adapter generated.
|
||||||
|
s := singleNodeSpec()
|
||||||
|
s.Profile.Args = []string{"--max-model-len", "32768"}
|
||||||
|
cmd := buildEngine(t, &VLLM{}, s)
|
||||||
|
|
||||||
|
i := slices.Index(cmd.Args, "--max-model-len")
|
||||||
|
j := slices.Index(cmd.Args, "--tensor-parallel-size")
|
||||||
|
if i < j {
|
||||||
|
t.Errorf("profile args came before generated ones: %v", cmd.Args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- validation ---
|
||||||
|
|
||||||
|
func h100(n int) []GPUInfo {
|
||||||
|
out := make([]GPUInfo, n)
|
||||||
|
for i := range out {
|
||||||
|
out[i] = GPUInfo{Index: i, Name: "NVIDIA H100 80GB HBM3", ComputeCapability: "9.0", MemoryTotalMB: 81559}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateAcceptsMatchingProfile(t *testing.T) {
|
||||||
|
p := Profile{Name: "ok", TP: 4, ModelPath: "/m"}
|
||||||
|
if err := (&VLLM{}).Validate(p, h100(4)); err != nil {
|
||||||
|
t.Errorf("a valid profile was rejected: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsMixedGPUTypes(t *testing.T) {
|
||||||
|
// The invariant from plan §1.1: a TP group never mixes GPU types.
|
||||||
|
gpus := h100(2)
|
||||||
|
gpus[1].Name = "NVIDIA A100-SXM4-40GB"
|
||||||
|
|
||||||
|
err := (&VLLM{}).Validate(Profile{Name: "mixed", TP: 2, ModelPath: "/m"}, gpus)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "may not mix GPU types") {
|
||||||
|
t.Errorf("mixed GPU types accepted: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsFP8OnOldHardware(t *testing.T) {
|
||||||
|
gpus := h100(4)
|
||||||
|
for i := range gpus {
|
||||||
|
gpus[i].Name = "NVIDIA A100-SXM4-80GB"
|
||||||
|
gpus[i].ComputeCapability = "8.0"
|
||||||
|
}
|
||||||
|
p := Profile{Name: "fp8", TP: 4, ModelPath: "/m", Args: []string{"--quantization", "fp8"}}
|
||||||
|
|
||||||
|
err := (&VLLM{}).Validate(p, gpus)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "compute capability") {
|
||||||
|
t.Errorf("fp8 accepted on compute capability 8.0: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The same profile is fine on Hopper.
|
||||||
|
if err := (&VLLM{}).Validate(p, h100(4)); err != nil {
|
||||||
|
t.Errorf("fp8 rejected on H100: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsInconsistentParallelism(t *testing.T) {
|
||||||
|
// 2 nodes x 8 GPUs cannot serve TP 4.
|
||||||
|
p := Profile{Name: "bad", TP: 4, Nodes: 2, GPUsPerNode: 8, ModelPath: "/m"}
|
||||||
|
if err := (&VLLM{}).Validate(p, h100(8)); err == nil {
|
||||||
|
t.Error("inconsistent node/GPU/TP arithmetic accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsTooFewGPUs(t *testing.T) {
|
||||||
|
if err := (&VLLM{}).Validate(Profile{Name: "big", TP: 8, ModelPath: "/m"}, h100(4)); err == nil {
|
||||||
|
t.Error("a TP 8 profile was accepted on a 4-GPU node")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRequiresModelPath(t *testing.T) {
|
||||||
|
if err := (&VLLM{}).Validate(Profile{Name: "nopath", TP: 1}, h100(1)); err == nil {
|
||||||
|
t.Error("a profile without a model path was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRegistry(t *testing.T) {
|
||||||
|
r := NewRegistry(Config{})
|
||||||
|
if got := r.Names(); !slices.Equal(got, []string{"llamacpp", "sglang", "sim", "vllm"}) {
|
||||||
|
t.Errorf("registry contents: %v", got)
|
||||||
|
}
|
||||||
|
if _, err := r.Get("vllm"); err != nil {
|
||||||
|
t.Errorf("vllm missing: %v", err)
|
||||||
|
}
|
||||||
|
err := func() error { _, err := r.Get("nope"); return err }()
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "llamacpp, sglang, sim, vllm") {
|
||||||
|
t.Errorf("an unknown engine should list what is available: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestQuantizationOf(t *testing.T) {
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{"--quantization", "fp8"},
|
||||||
|
{"--quantization=fp8"},
|
||||||
|
{"--max-model-len", "100", "--quantization", "fp8"},
|
||||||
|
} {
|
||||||
|
if got := quantizationOf(args); got != "fp8" {
|
||||||
|
t.Errorf("quantizationOf(%v)=%q", args, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := quantizationOf([]string{"--quantization"}); got != "" {
|
||||||
|
t.Errorf("a dangling flag should yield nothing, got %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
package engines
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// LlamaCppConfig locates the llama.cpp installation.
|
||||||
|
type LlamaCppConfig struct {
|
||||||
|
// Bin is the llama-server binary. It is the server, not the CLI: the
|
||||||
|
// gateway talks OpenAI over HTTP and needs the one that serves it.
|
||||||
|
Bin string `koanf:"bin"`
|
||||||
|
// Parallel is how many requests a server handles at once, when a profile
|
||||||
|
// does not say. llama.cpp defaults to one, which would make every second
|
||||||
|
// request wait behind the first no matter what the router decided.
|
||||||
|
Parallel int `koanf:"parallel"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// LlamaCpp builds llama.cpp launch commands.
|
||||||
|
//
|
||||||
|
// It is the engine that makes a cluster without GPUs useful: llama.cpp runs a
|
||||||
|
// small model on CPU at a speed that is fine for testing the whole path, and it
|
||||||
|
// serves the same OpenAI API as the others, so nothing downstream can tell the
|
||||||
|
// difference.
|
||||||
|
type LlamaCpp struct{ cfg LlamaCppConfig }
|
||||||
|
|
||||||
|
// Name identifies the engine in the catalog.
|
||||||
|
func (*LlamaCpp) Name() string { return "llamacpp" }
|
||||||
|
|
||||||
|
// HealthPath is llama.cpp's readiness endpoint. It returns 503 while the model
|
||||||
|
// is still loading, which is exactly what the supervisor needs.
|
||||||
|
func (*LlamaCpp) HealthPath() string { return "/health" }
|
||||||
|
|
||||||
|
// MetricsPath is llama.cpp's Prometheus endpoint, which only exists when the
|
||||||
|
// server was started with --metrics. Commands always passes it.
|
||||||
|
func (*LlamaCpp) MetricsPath() string { return "/metrics" }
|
||||||
|
|
||||||
|
// ServesHTTP reports whether this rank answers the API. Every llama.cpp shard
|
||||||
|
// does, because a llama.cpp replica is always exactly one shard.
|
||||||
|
func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true }
|
||||||
|
|
||||||
|
// Validate checks the profile against the GPUs it would run on.
|
||||||
|
//
|
||||||
|
// The two refusals here are about not pretending: llama.cpp has no equivalent
|
||||||
|
// of a tensor-parallel group spanning machines, and its multi-GPU support is a
|
||||||
|
// different mechanism (--split-mode, --tensor-split) that this adapter does not
|
||||||
|
// drive. Accepting either would produce a replica that starts and then serves
|
||||||
|
// from one machine while the catalog believes it has several.
|
||||||
|
func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error {
|
||||||
|
if err := validateCommon(p, gpus); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if p.Nodes > 1 {
|
||||||
|
return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica",
|
||||||
|
p.Name, p.Nodes)
|
||||||
|
}
|
||||||
|
if p.TP > 1 {
|
||||||
|
return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+
|
||||||
|
"it splits a model across local GPUs with --tensor-split, which this adapter does not drive",
|
||||||
|
p.Name)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Commands returns the processes for one shard, which is always one process.
|
||||||
|
func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
|
||||||
|
bin := l.cfg.Bin
|
||||||
|
if bin == "" {
|
||||||
|
bin = "llama-server"
|
||||||
|
}
|
||||||
|
p := s.Profile
|
||||||
|
|
||||||
|
args := []string{
|
||||||
|
"--model", p.ModelPath,
|
||||||
|
"--host", "0.0.0.0",
|
||||||
|
"--port", strconv.Itoa(s.HTTPPort),
|
||||||
|
// Without this there is no /metrics, and an endpoint the router cannot
|
||||||
|
// measure is one it has to route to blind.
|
||||||
|
"--metrics",
|
||||||
|
}
|
||||||
|
// The model's own chat template rather than a generic one, which is what
|
||||||
|
// renders tool definitions in the form the model was trained to read.
|
||||||
|
//
|
||||||
|
// Pinned rather than relied on: build 10991 has it on by default, older
|
||||||
|
// builds do not, and an engine upgrade should not be able to change how
|
||||||
|
// prompts are built underneath a running fleet. Passing it explicitly was
|
||||||
|
// a no-op on the build here — it was added while chasing tool calls that
|
||||||
|
// came back as text, and did not fix that, which turned out to be the
|
||||||
|
// model rather than the template.
|
||||||
|
if !hasFlag(p.Args, "--jinja", "--no-jinja") {
|
||||||
|
args = append(args, "--jinja")
|
||||||
|
}
|
||||||
|
if p.ServedModelName != "" {
|
||||||
|
args = append(args, "--alias", p.ServedModelName)
|
||||||
|
}
|
||||||
|
if !hasFlag(p.Args, "--parallel", "-np") {
|
||||||
|
n := l.cfg.Parallel
|
||||||
|
if n <= 0 {
|
||||||
|
n = 4
|
||||||
|
}
|
||||||
|
args = append(args, "--parallel", strconv.Itoa(n))
|
||||||
|
}
|
||||||
|
// With GPUs assigned, offload everything; llama.cpp keeps the model on the
|
||||||
|
// CPU otherwise, which is not what a GPU placement asked for. Without them
|
||||||
|
// the flag is omitted entirely rather than set to zero, so a build with no
|
||||||
|
// GPU support does not see an argument it rejects.
|
||||||
|
if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") {
|
||||||
|
args = append(args, "--n-gpu-layers", "999")
|
||||||
|
}
|
||||||
|
args = append(args, p.Args...)
|
||||||
|
|
||||||
|
return []Command{{
|
||||||
|
Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
|
||||||
|
}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// hasFlag reports whether the profile already sets one of these flags, in
|
||||||
|
// either "--flag value" or "--flag=value" form.
|
||||||
|
func hasFlag(args []string, names ...string) bool {
|
||||||
|
for _, a := range args {
|
||||||
|
for _, n := range names {
|
||||||
|
if a == n || strings.HasPrefix(a, n+"=") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,189 @@
|
|||||||
|
package engines
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func llamaSpec(gpus []int) ShardSpec {
|
||||||
|
return ShardSpec{
|
||||||
|
ReplicaID: "rep-1", ShardID: "sh-1", Rank: 0, NNodes: 1,
|
||||||
|
GPUIndices: gpus, HTTPPort: 8000, MgmtInterface: "eth0",
|
||||||
|
Profile: Profile{
|
||||||
|
Name: "cpu", Engine: "llamacpp", TP: 1,
|
||||||
|
ModelPath: "/models/qwen.gguf", ServedModelName: "qwen-0.5b",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLlamaCppCommand(t *testing.T) {
|
||||||
|
cmds, err := (&LlamaCpp{}).Commands(llamaSpec(nil))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(cmds) != 1 {
|
||||||
|
t.Fatalf("a llama.cpp shard is one process, got %d", len(cmds))
|
||||||
|
}
|
||||||
|
c := cmds[0]
|
||||||
|
if c.Path != "llama-server" {
|
||||||
|
t.Errorf("path %q", c.Path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// --metrics is not optional: llama-server serves no /metrics without it, and
|
||||||
|
// an endpoint the router cannot measure is one it routes to blind.
|
||||||
|
if !slices.Contains(c.Args, "--metrics") {
|
||||||
|
t.Error("the server was started without --metrics")
|
||||||
|
}
|
||||||
|
if v, _ := flagValue(c.Args, "--model"); v != "/models/qwen.gguf" {
|
||||||
|
t.Errorf("--model %q", v)
|
||||||
|
}
|
||||||
|
if v, _ := flagValue(c.Args, "--port"); v != "8000" {
|
||||||
|
t.Errorf("--port %q", v)
|
||||||
|
}
|
||||||
|
if v, _ := flagValue(c.Args, "--alias"); v != "qwen-0.5b" {
|
||||||
|
t.Errorf("--alias %q", v)
|
||||||
|
}
|
||||||
|
// llama.cpp serves one request at a time unless told otherwise, which would
|
||||||
|
// make every second request wait no matter what the router decided.
|
||||||
|
if v, ok := flagValue(c.Args, "--parallel"); !ok || v == "1" {
|
||||||
|
t.Errorf("--parallel %q", v)
|
||||||
|
}
|
||||||
|
if slices.Contains(c.Args, "--n-gpu-layers") {
|
||||||
|
t.Error("a CPU placement was given GPU layers")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLlamaCppOffloadsWhenGivenGPUs(t *testing.T) {
|
||||||
|
cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
|
||||||
|
if v, ok := flagValue(cmds[0].Args, "--n-gpu-layers"); !ok || v != "999" {
|
||||||
|
t.Errorf("a GPU placement should offload the model, got %q", v)
|
||||||
|
}
|
||||||
|
// The shard must still be pinned to its own card.
|
||||||
|
if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=2") {
|
||||||
|
t.Error("the shard was not pinned to its GPU")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileArgumentsWinOverDefaults(t *testing.T) {
|
||||||
|
// The escape hatch has to actually work, or an operator tuning slot count
|
||||||
|
// ends up with two --parallel flags and whichever llama.cpp happens to
|
||||||
|
// prefer.
|
||||||
|
s := llamaSpec(nil)
|
||||||
|
s.Profile.Args = []string{"--parallel", "16", "--ctx-size", "4096"}
|
||||||
|
|
||||||
|
cmds, _ := (&LlamaCpp{}).Commands(s)
|
||||||
|
n := 0
|
||||||
|
for _, a := range cmds[0].Args {
|
||||||
|
if a == "--parallel" {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n != 1 {
|
||||||
|
t.Errorf("--parallel appears %d times", n)
|
||||||
|
}
|
||||||
|
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "16" {
|
||||||
|
t.Errorf("the profile's value lost: %q", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestShortFormArgumentsAreRecognized(t *testing.T) {
|
||||||
|
s := llamaSpec([]int{0})
|
||||||
|
s.Profile.Args = []string{"-np", "8", "-ngl", "20"}
|
||||||
|
|
||||||
|
cmds, _ := (&LlamaCpp{}).Commands(s)
|
||||||
|
for _, flag := range []string{"--parallel", "--n-gpu-layers"} {
|
||||||
|
if slices.Contains(cmds[0].Args, flag) {
|
||||||
|
t.Errorf("%s was added although the profile set its short form", flag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLlamaCppRefusesWhatItCannotDo(t *testing.T) {
|
||||||
|
// Accepting either of these would produce a replica that starts and then
|
||||||
|
// serves from one machine while the catalog believes it has several.
|
||||||
|
multi := Profile{Name: "multi", Engine: "llamacpp", TP: 1, Nodes: 2, GPUsPerNode: 1, ModelPath: "/m"}
|
||||||
|
if err := (&LlamaCpp{}).Validate(multi, nil); err == nil {
|
||||||
|
t.Error("a multi-node llama.cpp profile was accepted")
|
||||||
|
}
|
||||||
|
|
||||||
|
tp := Profile{Name: "tp", Engine: "llamacpp", TP: 2, GPUsPerNode: 2, ModelPath: "/m"}
|
||||||
|
if err := (&LlamaCpp{}).Validate(tp, h100(2)); err == nil {
|
||||||
|
t.Error("a tensor-parallel llama.cpp profile was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPUProfileValidates(t *testing.T) {
|
||||||
|
p := Profile{Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf"}
|
||||||
|
if err := (&LlamaCpp{}).Validate(p, nil); err != nil {
|
||||||
|
t.Errorf("a CPU profile on a node with no GPUs was rejected: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLlamaCppAlwaysServesHTTP(t *testing.T) {
|
||||||
|
// Unlike the others there is no worker rank to get this wrong for.
|
||||||
|
if !(&LlamaCpp{}).ServesHTTP(llamaSpec(nil)) {
|
||||||
|
t.Error("a llama.cpp shard must answer the API")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBinaryIsConfigurable(t *testing.T) {
|
||||||
|
l := &LlamaCpp{cfg: LlamaCppConfig{Bin: "/opt/llama.cpp/bin/llama-server", Parallel: 2}}
|
||||||
|
cmds, _ := l.Commands(llamaSpec(nil))
|
||||||
|
if cmds[0].Path != "/opt/llama.cpp/bin/llama-server" {
|
||||||
|
t.Errorf("path %q", cmds[0].Path)
|
||||||
|
}
|
||||||
|
if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "2" {
|
||||||
|
t.Errorf("--parallel %q", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnknownEngineNamesLlamaCpp(t *testing.T) {
|
||||||
|
_, err := NewRegistry(Config{}).Get("llama.cpp")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "llamacpp") {
|
||||||
|
t.Errorf("the suggestion should name the real engine: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// flagValue reads a "--flag value" pair out of a command line.
|
||||||
|
func flagValue(args []string, flag string) (string, bool) {
|
||||||
|
for i, a := range args {
|
||||||
|
if a == flag && i+1 < len(args) {
|
||||||
|
return args[i+1], true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLlamaCppUsesTheModelsChatTemplate(t *testing.T) {
|
||||||
|
// Pinned so an engine upgrade cannot change how prompts are built under a
|
||||||
|
// running fleet: build 10991 defaults to jinja, older builds do not.
|
||||||
|
//
|
||||||
|
// Not a fix for tool calling, though it was added while chasing that. On
|
||||||
|
// this build the flag is a no-op and the behaviour is identical with and
|
||||||
|
// without it.
|
||||||
|
l := &LlamaCpp{}
|
||||||
|
cmds, err := l.Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: "/m.gguf"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !slices.Contains(cmds[0].Args, "--jinja") {
|
||||||
|
t.Errorf("--jinja is missing; tool calls will come back as text: %v", cmds[0].Args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLlamaCppLetsAProfileTurnTheTemplateOff(t *testing.T) {
|
||||||
|
// A model whose bundled template is broken has to be serviceable without
|
||||||
|
// editing the adapter.
|
||||||
|
l := &LlamaCpp{}
|
||||||
|
cmds, err := l.Commands(ShardSpec{
|
||||||
|
HTTPPort: 8080,
|
||||||
|
Profile: Profile{ModelPath: "/m.gguf", Args: []string{"--no-jinja"}},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if slices.Contains(cmds[0].Args, "--jinja") {
|
||||||
|
t.Errorf("the profile said --no-jinja and got both: %v", cmds[0].Args)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package engines
|
||||||
|
|
||||||
|
import "strconv"
|
||||||
|
|
||||||
|
// SGLangConfig locates the SGLang installation.
|
||||||
|
type SGLangConfig struct {
|
||||||
|
// Python is the interpreter of the SGLang virtualenv. SGLang is launched as
|
||||||
|
// a module rather than a console script, which is the documented entry
|
||||||
|
// point and avoids depending on the script being on PATH.
|
||||||
|
Python string `koanf:"python"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SGLang builds SGLang launch commands.
|
||||||
|
type SGLang struct{ cfg SGLangConfig }
|
||||||
|
|
||||||
|
// Name identifies the engine in the catalog.
|
||||||
|
func (*SGLang) Name() string { return "sglang" }
|
||||||
|
|
||||||
|
// HealthPath is SGLang's readiness endpoint.
|
||||||
|
func (*SGLang) HealthPath() string { return "/health" }
|
||||||
|
|
||||||
|
// MetricsPath is SGLang's Prometheus endpoint.
|
||||||
|
func (*SGLang) MetricsPath() string { return "/metrics" }
|
||||||
|
|
||||||
|
// Validate checks the profile against the GPUs it would run on.
|
||||||
|
func (*SGLang) Validate(p Profile, gpus []GPUInfo) error {
|
||||||
|
if err := validateCommon(p, gpus); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if q := quantizationOf(p.Args); q == "fp8" {
|
||||||
|
if err := requireComputeCapability(p, gpus, 8.9, "fp8 quantization"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServesHTTP reports whether this rank answers the API.
|
||||||
|
//
|
||||||
|
// Every SGLang rank runs the same launcher, but only node-rank 0 binds the HTTP
|
||||||
|
// port; the others exist to contribute GPUs to the tensor-parallel group.
|
||||||
|
func (*SGLang) ServesHTTP(s ShardSpec) bool { return s.Leader() }
|
||||||
|
|
||||||
|
// Commands returns the processes for one shard.
|
||||||
|
//
|
||||||
|
// SGLang supports multi-node natively, so there is only ever one process: every
|
||||||
|
// rank runs the same launcher with its own rank number and they rendezvous at
|
||||||
|
// the leader's dist-init address.
|
||||||
|
func (g *SGLang) Commands(s ShardSpec) ([]Command, error) {
|
||||||
|
c, err := g.serverCommand(s)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []Command{c}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
|
||||||
|
python := g.cfg.Python
|
||||||
|
if python == "" {
|
||||||
|
python = "python3"
|
||||||
|
}
|
||||||
|
p := s.Profile
|
||||||
|
|
||||||
|
args := []string{"-m", "sglang.launch_server",
|
||||||
|
"--model-path", p.ModelPath,
|
||||||
|
"--host", "0.0.0.0",
|
||||||
|
"--port", strconv.Itoa(s.HTTPPort),
|
||||||
|
"--tp", strconv.Itoa(p.TP),
|
||||||
|
}
|
||||||
|
if p.ServedModelName != "" {
|
||||||
|
args = append(args, "--served-model-name", p.ServedModelName)
|
||||||
|
}
|
||||||
|
if s.NNodes > 1 {
|
||||||
|
args = append(args,
|
||||||
|
"--nnodes", strconv.Itoa(s.NNodes),
|
||||||
|
"--node-rank", strconv.Itoa(s.Rank),
|
||||||
|
"--dist-init-addr", s.LeaderAddr+":"+strconv.Itoa(s.LeaderPort),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
args = append(args, p.Args...)
|
||||||
|
|
||||||
|
return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
package engines
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SimConfig locates the simulated engine.
|
||||||
|
type SimConfig struct {
|
||||||
|
// Bin is the llmgw binary itself; the engine is one of its subcommands.
|
||||||
|
// Empty means whatever started this agent, which is right almost always.
|
||||||
|
Bin string `koanf:"bin"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sim launches llmgw's own simulated engine.
|
||||||
|
//
|
||||||
|
// It is in the production registry on purpose. A cluster can be brought up and
|
||||||
|
// exercised end to end — placement, fabrics, multi-node sequencing, failure and
|
||||||
|
// rebuild — on machines that have no GPUs, which is the only way most of that
|
||||||
|
// path gets tested at all. The multi-node launch sequence went four phases
|
||||||
|
// without ever running, because the only things exercising it were stub
|
||||||
|
// processes that exercised the command line and nothing else.
|
||||||
|
//
|
||||||
|
// Nothing about it pretends to be real: the model name it serves is whatever it
|
||||||
|
// is told, the tokens are invented, and it says so in its own logs.
|
||||||
|
type Sim struct{ cfg SimConfig }
|
||||||
|
|
||||||
|
// Name identifies the engine in the catalog.
|
||||||
|
func (*Sim) Name() string { return "sim" }
|
||||||
|
|
||||||
|
// HealthPath is the simulated engine's readiness endpoint.
|
||||||
|
func (*Sim) HealthPath() string { return "/health" }
|
||||||
|
|
||||||
|
// MetricsPath is its Prometheus endpoint, in vLLM's vocabulary.
|
||||||
|
func (*Sim) MetricsPath() string { return "/metrics" }
|
||||||
|
|
||||||
|
// ServesHTTP reports whether this rank answers the API.
|
||||||
|
//
|
||||||
|
// Only the leader, exactly as vLLM and SGLang behave — which is the behaviour
|
||||||
|
// this engine exists to let us test.
|
||||||
|
func (*Sim) ServesHTTP(s ShardSpec) bool { return s.Leader() }
|
||||||
|
|
||||||
|
// Validate accepts anything the arithmetic allows.
|
||||||
|
//
|
||||||
|
// No compute-capability checks and no quantization rules: there is no hardware
|
||||||
|
// to be wrong about, and inventing refusals would make this a worse stand-in
|
||||||
|
// for the engines it stands in for.
|
||||||
|
func (*Sim) Validate(p Profile, gpus []GPUInfo) error {
|
||||||
|
return validateCommon(p, gpus)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Commands returns the single process that makes up one shard.
|
||||||
|
func (s *Sim) Commands(spec ShardSpec) ([]Command, error) {
|
||||||
|
bin := s.cfg.Bin
|
||||||
|
if bin == "" {
|
||||||
|
exe, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("locating the llmgw binary for the sim engine: %w", err)
|
||||||
|
}
|
||||||
|
bin = exe
|
||||||
|
}
|
||||||
|
p := spec.Profile
|
||||||
|
|
||||||
|
args := []string{"sim-engine",
|
||||||
|
"--listen", "0.0.0.0:" + strconv.Itoa(spec.HTTPPort),
|
||||||
|
"--model", p.ServedModelName,
|
||||||
|
}
|
||||||
|
if spec.NNodes > 1 {
|
||||||
|
args = append(args,
|
||||||
|
"--nnodes", strconv.Itoa(spec.NNodes),
|
||||||
|
"--node-rank", strconv.Itoa(spec.Rank),
|
||||||
|
"--dist-init-addr", spec.LeaderAddr+":"+strconv.Itoa(spec.LeaderPort),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
args = append(args, p.Args...)
|
||||||
|
|
||||||
|
return []Command{{
|
||||||
|
Name: "sim-engine", Path: bin, Args: args,
|
||||||
|
Env: materializeEnv(baseEnv(spec)),
|
||||||
|
// Workers dial the leader, so the leader has to be listening first.
|
||||||
|
// Both ranks express the same wait because both know the same address.
|
||||||
|
WaitForLeaderPort: spec.NNodes > 1 && !spec.Leader(),
|
||||||
|
}}, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package engines
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func simSpec(rank, nnodes int) ShardSpec {
|
||||||
|
return ShardSpec{
|
||||||
|
ReplicaID: "rep-1", ShardID: "sh-1", Rank: rank, NNodes: nnodes,
|
||||||
|
LeaderAddr: "10.0.0.5", LeaderPort: 20001,
|
||||||
|
GPUIndices: []int{0, 1}, HTTPPort: 20000, MgmtInterface: "eth0",
|
||||||
|
HCAs: []string{"mlx5_0", "mlx5_1"},
|
||||||
|
Profile: Profile{
|
||||||
|
Name: "tp4", Engine: "sim", TP: 4, GPUsPerNode: 2, Nodes: nnodes,
|
||||||
|
ModelPath: "/models/x", ServedModelName: "big-model",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOnlyTheLeaderServes(t *testing.T) {
|
||||||
|
// The behaviour that made replicas fail while they were coming up perfectly
|
||||||
|
// well: health checking a worker's port, which never opens.
|
||||||
|
if !(&Sim{}).ServesHTTP(simSpec(0, 2)) {
|
||||||
|
t.Error("the leader does not serve")
|
||||||
|
}
|
||||||
|
if (&Sim{}).ServesHTTP(simSpec(1, 2)) {
|
||||||
|
t.Error("a worker claims to serve")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMultiNodeShardCarriesItsRendezvous(t *testing.T) {
|
||||||
|
cmds, err := (&Sim{cfg: SimConfig{Bin: "/usr/local/bin/llmgw"}}).Commands(simSpec(1, 2))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(cmds) != 1 {
|
||||||
|
t.Fatalf("%d commands", len(cmds))
|
||||||
|
}
|
||||||
|
c := cmds[0]
|
||||||
|
|
||||||
|
if c.Args[0] != "sim-engine" {
|
||||||
|
t.Errorf("the engine is a subcommand of llmgw: %v", c.Args)
|
||||||
|
}
|
||||||
|
for flag, want := range map[string]string{
|
||||||
|
"--nnodes": "2", "--node-rank": "1", "--dist-init-addr": "10.0.0.5:20001",
|
||||||
|
} {
|
||||||
|
if got, _ := flagValue(c.Args, flag); got != want {
|
||||||
|
t.Errorf("%s = %q, want %q", flag, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A worker must wait for the leader's port, or the launch is a race.
|
||||||
|
if !c.WaitForLeaderPort {
|
||||||
|
t.Error("a worker does not wait for the leader")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheLeaderDoesNotWaitForItself(t *testing.T) {
|
||||||
|
cmds, _ := (&Sim{cfg: SimConfig{Bin: "x"}}).Commands(simSpec(0, 2))
|
||||||
|
if cmds[0].WaitForLeaderPort {
|
||||||
|
t.Error("the leader waits for a port it is about to open")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestASingleNodeShardCarriesNoRendezvous(t *testing.T) {
|
||||||
|
cmds, _ := (&Sim{cfg: SimConfig{Bin: "x"}}).Commands(simSpec(0, 1))
|
||||||
|
for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {
|
||||||
|
if slices.Contains(cmds[0].Args, flag) {
|
||||||
|
t.Errorf("a single-node shard was given %s", flag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheShardIsStillPinnedToItsFabric(t *testing.T) {
|
||||||
|
// The simulated engine does not use NCCL, but the environment is generated
|
||||||
|
// by the same code that generates it for the ones that do — so a cluster
|
||||||
|
// exercised with this engine is exercising that too.
|
||||||
|
cmds, _ := (&Sim{cfg: SimConfig{Bin: "x"}}).Commands(simSpec(1, 2))
|
||||||
|
env := cmds[0].Env
|
||||||
|
for _, want := range []string{
|
||||||
|
"NCCL_IB_HCA=mlx5_0,mlx5_1", "NCCL_SOCKET_IFNAME=eth0",
|
||||||
|
"CUDA_VISIBLE_DEVICES=0,1", "NCCL_IB_DISABLE=0",
|
||||||
|
} {
|
||||||
|
if !slices.Contains(env, want) {
|
||||||
|
t.Errorf("the environment is missing %s", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,149 @@
|
|||||||
|
package engines
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
)
|
||||||
|
|
||||||
|
// VLLMConfig locates the vLLM installation and picks how it is launched.
|
||||||
|
type VLLMConfig struct {
|
||||||
|
// Bin is the vllm executable.
|
||||||
|
Bin string `koanf:"bin"`
|
||||||
|
// LaunchMode selects the multi-node mechanism: "ray" or "mp".
|
||||||
|
//
|
||||||
|
// Which one works depends on the vLLM version, and the flags have changed
|
||||||
|
// more than once (plan §14). Both paths exist so a cluster can pick the one
|
||||||
|
// its version supports instead of waiting for a code change.
|
||||||
|
LaunchMode string `koanf:"launch_mode"`
|
||||||
|
// RayBin is used when LaunchMode is "ray".
|
||||||
|
RayBin string `koanf:"ray_bin"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// VLLM builds vLLM launch commands.
|
||||||
|
type VLLM struct{ cfg VLLMConfig }
|
||||||
|
|
||||||
|
// Name identifies the engine in the catalog.
|
||||||
|
func (*VLLM) Name() string { return "vllm" }
|
||||||
|
|
||||||
|
// HealthPath is vLLM's readiness endpoint.
|
||||||
|
func (*VLLM) HealthPath() string { return "/health" }
|
||||||
|
|
||||||
|
// MetricsPath is vLLM's Prometheus endpoint.
|
||||||
|
func (*VLLM) MetricsPath() string { return "/metrics" }
|
||||||
|
|
||||||
|
// Validate checks the profile against the GPUs it would run on.
|
||||||
|
func (*VLLM) Validate(p Profile, gpus []GPUInfo) error {
|
||||||
|
if err := validateCommon(p, gpus); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// FP8 needs Hopper or Ada; on anything older vLLM either refuses or falls
|
||||||
|
// back silently to a slower path, and a silent fallback is worse.
|
||||||
|
if q := quantizationOf(p.Args); q == "fp8" {
|
||||||
|
if err := requireComputeCapability(p, gpus, 8.9, "fp8 quantization"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServesHTTP reports whether this rank answers the API.
|
||||||
|
//
|
||||||
|
// Under Ray a worker runs nothing but the Ray join, and under multiprocessing
|
||||||
|
// only the driver binds the port. Either way it is the leader alone.
|
||||||
|
func (*VLLM) ServesHTTP(s ShardSpec) bool { return s.Leader() }
|
||||||
|
|
||||||
|
// Commands returns the processes for one shard, in start order.
|
||||||
|
//
|
||||||
|
// Single node, or multi-node under multiprocessing: one server process. Under
|
||||||
|
// Ray: the leader starts a head and then the server, a worker starts only the
|
||||||
|
// join. A Ray worker running no vLLM process at all is correct — the cluster
|
||||||
|
// gives the leader its GPUs.
|
||||||
|
func (v *VLLM) Commands(s ShardSpec) ([]Command, error) {
|
||||||
|
ray, useRay := v.rayCommand(s)
|
||||||
|
if useRay && !s.Leader() {
|
||||||
|
// A worker is the Ray join and nothing else. It blocks, so its exit is
|
||||||
|
// the shard's exit, which is what the supervisor needs.
|
||||||
|
return []Command{ray}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
server, err := v.serverCommand(s)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if useRay {
|
||||||
|
return []Command{ray, server}, nil
|
||||||
|
}
|
||||||
|
return []Command{server}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// serverCommand produces the vLLM server process.
|
||||||
|
func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
|
||||||
|
bin := v.cfg.Bin
|
||||||
|
if bin == "" {
|
||||||
|
bin = "vllm"
|
||||||
|
}
|
||||||
|
p := s.Profile
|
||||||
|
|
||||||
|
args := []string{"serve", p.ModelPath,
|
||||||
|
"--host", "0.0.0.0",
|
||||||
|
"--port", strconv.Itoa(s.HTTPPort),
|
||||||
|
"--tensor-parallel-size", strconv.Itoa(p.TP),
|
||||||
|
}
|
||||||
|
if p.PP > 1 {
|
||||||
|
args = append(args, "--pipeline-parallel-size", strconv.Itoa(p.PP))
|
||||||
|
}
|
||||||
|
if p.ServedModelName != "" {
|
||||||
|
args = append(args, "--served-model-name", p.ServedModelName)
|
||||||
|
}
|
||||||
|
|
||||||
|
env := baseEnv(s)
|
||||||
|
|
||||||
|
if s.NNodes > 1 {
|
||||||
|
switch v.cfg.LaunchMode {
|
||||||
|
case "", "ray":
|
||||||
|
// Ray is started separately by the supervisor; vLLM then finds the
|
||||||
|
// cluster through the address in the environment.
|
||||||
|
env["VLLM_HOST_IP"] = s.LeaderAddr
|
||||||
|
env["RAY_ADDRESS"] = fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort)
|
||||||
|
args = append(args, "--distributed-executor-backend", "ray")
|
||||||
|
case "mp":
|
||||||
|
env["VLLM_HOST_IP"] = s.LeaderAddr
|
||||||
|
env["VLLM_PORT"] = strconv.Itoa(s.LeaderPort)
|
||||||
|
args = append(args, "--distributed-executor-backend", "mp")
|
||||||
|
default:
|
||||||
|
return Command{}, fmt.Errorf("vllm: unknown launch_mode %q (want \"ray\" or \"mp\")", v.cfg.LaunchMode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Profile arguments last, so an operator can override anything above.
|
||||||
|
args = append(args, p.Args...)
|
||||||
|
|
||||||
|
return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// rayCommand returns the command that starts or joins the Ray cluster, or
|
||||||
|
// ok=false when the launch mode does not use Ray.
|
||||||
|
func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) {
|
||||||
|
if s.NNodes <= 1 || (v.cfg.LaunchMode != "" && v.cfg.LaunchMode != "ray") {
|
||||||
|
return Command{}, false
|
||||||
|
}
|
||||||
|
bin := v.cfg.RayBin
|
||||||
|
if bin == "" {
|
||||||
|
bin = "ray"
|
||||||
|
}
|
||||||
|
|
||||||
|
args := []string{"start", "--block"}
|
||||||
|
if s.Leader() {
|
||||||
|
args = append(args, "--head", "--port", strconv.Itoa(s.LeaderPort))
|
||||||
|
} else {
|
||||||
|
args = append(args, "--address", fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort))
|
||||||
|
}
|
||||||
|
args = append(args, "--num-gpus", strconv.Itoa(len(s.GPUIndices)))
|
||||||
|
|
||||||
|
return Command{
|
||||||
|
Name: "ray", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
|
||||||
|
// The head must be listening before anything else happens: a worker
|
||||||
|
// cannot join, and the leader's server cannot claim GPUs, until it is.
|
||||||
|
WaitForLeaderPort: true,
|
||||||
|
}, true
|
||||||
|
}
|
||||||
@@ -0,0 +1,317 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/net/http2"
|
||||||
|
"golang.org/x/net/http2/h2c"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||||
|
)
|
||||||
|
|
||||||
|
// pki issues a cluster's certificates into a temporary directory and returns
|
||||||
|
// the configurations the control plane and one agent would load.
|
||||||
|
func pki(t *testing.T, nodeID string) (server, client tlsconf.Config) {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
ca, err := tlsconf.NewCA("test CA")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := ca.Write(dir); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
srv, err := ca.IssueServer("control-plane", []string{"127.0.0.1"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := srv.Write(dir, "server"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cli, err := ca.IssueClient(nodeID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := cli.Write(dir, "agent"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
caPath := filepath.Join(dir, "ca.pem")
|
||||||
|
return tlsconf.Config{CA: caPath, Cert: filepath.Join(dir, "server.pem"), Key: filepath.Join(dir, "server-key.pem")},
|
||||||
|
tlsconf.Config{CA: caPath, Cert: filepath.Join(dir, "agent.pem"), Key: filepath.Join(dir, "agent-key.pem")}
|
||||||
|
}
|
||||||
|
|
||||||
|
// startTLS runs a real service behind a real TLS listener, with identity
|
||||||
|
// required — the deployed configuration rather than an approximation of it.
|
||||||
|
func startTLS(t *testing.T, serverTLS tlsconf.Config) (*agentsvc.Service, *inventory.Store, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
inv := inventory.NewStore()
|
||||||
|
svc := agentsvc.New(inv, newRecorder(), true, slog.New(slog.DiscardHandler))
|
||||||
|
path, handler := svc.Handler()
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
mux.Handle(path, handler)
|
||||||
|
|
||||||
|
tc, err := serverTLS.ServerConfig()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
raw, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
srv := httptest.NewUnstartedServer(mux)
|
||||||
|
srv.Listener = raw
|
||||||
|
srv.TLS = tc
|
||||||
|
srv.StartTLS()
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
|
||||||
|
return svc, inv, "https://" + raw.Addr().String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func runAgent(t *testing.T, cfg Config) *Agent {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
cfg.MgmtAddr = "127.0.0.1"
|
||||||
|
cfg.HeartbeatInterval = 50 * time.Millisecond
|
||||||
|
cfg.ReconnectBase = 20 * time.Millisecond
|
||||||
|
cfg.ReconnectMax = 100 * time.Millisecond
|
||||||
|
cfg.Discovery.SkipGPUs = true
|
||||||
|
cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent")
|
||||||
|
cfg.Superv = supervisor.Config{
|
||||||
|
WorkDir: t.TempDir(), LogLines: 64,
|
||||||
|
StopTimeout: 2 * time.Second, HealthInterval: 20 * time.Millisecond,
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err := Client(cfg.TLS, 5*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ag, err := New(cfg, client, slog.New(slog.DiscardHandler))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
go func() { _ = ag.Run(ctx) }()
|
||||||
|
t.Cleanup(func() {
|
||||||
|
cancel()
|
||||||
|
ag.Supervisor().StopAll(context.Background())
|
||||||
|
})
|
||||||
|
return ag
|
||||||
|
}
|
||||||
|
|
||||||
|
func registered(svc *agentsvc.Service, inv *inventory.Store, nodeID string) func() bool {
|
||||||
|
return func() bool {
|
||||||
|
_, ok := inv.Get(nodeID)
|
||||||
|
return ok && len(svc.ConnectedNodes()) == 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func eventually(t *testing.T, limit time.Duration, pred func() bool) bool {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(limit)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if pred() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAgentRegistersOverMutualTLS(t *testing.T) {
|
||||||
|
serverTLS, clientTLS := pki(t, "llmgw-node01")
|
||||||
|
svc, inv, url := startTLS(t, serverTLS)
|
||||||
|
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.NodeID = "llmgw-node01"
|
||||||
|
cfg.Server = url
|
||||||
|
cfg.TLS = clientTLS
|
||||||
|
runAgent(t, cfg)
|
||||||
|
|
||||||
|
if !eventually(t, 10*time.Second, registered(svc, inv, "llmgw-node01")) {
|
||||||
|
t.Fatal("the agent never registered over mutual TLS")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNodeCannotClaimAnotherNodesIdentity(t *testing.T) {
|
||||||
|
// The point of the whole exercise. node01 holds a valid certificate — it is
|
||||||
|
// a legitimate member of the cluster — and still cannot register as node02,
|
||||||
|
// because the id it claims is checked against the one it can prove.
|
||||||
|
serverTLS, clientTLS := pki(t, "llmgw-node01")
|
||||||
|
svc, inv, url := startTLS(t, serverTLS)
|
||||||
|
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.NodeID = "llmgw-node02"
|
||||||
|
cfg.Server = url
|
||||||
|
cfg.TLS = clientTLS
|
||||||
|
runAgent(t, cfg)
|
||||||
|
|
||||||
|
// The agent reconnects on failure, so this has to stay false throughout
|
||||||
|
// rather than merely start false.
|
||||||
|
if eventually(t, 2*time.Second, func() bool {
|
||||||
|
_, ok := inv.Get("llmgw-node02")
|
||||||
|
return ok || len(svc.ConnectedNodes()) > 0
|
||||||
|
}) {
|
||||||
|
t.Fatal("an agent registered under a node id its certificate does not authorize")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnauthenticatedAgentIsRefused(t *testing.T) {
|
||||||
|
// The defensive half of the check: identity is required, but the connection
|
||||||
|
// carried none. In the shipped wiring TLS and this flag are tied together,
|
||||||
|
// so reaching here means someone untied them — and the answer must still be
|
||||||
|
// refusal rather than a silently trusted node id.
|
||||||
|
inv := inventory.NewStore()
|
||||||
|
svc := agentsvc.New(inv, newRecorder(), true, slog.New(slog.DiscardHandler))
|
||||||
|
path, handler := svc.Handler()
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
mux.Handle(path, handler)
|
||||||
|
|
||||||
|
srv := httptest.NewServer(h2c.NewHandler(mux, &http2.Server{}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.NodeID = "llmgw-node01"
|
||||||
|
cfg.Server = srv.URL
|
||||||
|
runAgent(t, cfg)
|
||||||
|
|
||||||
|
if eventually(t, 2*time.Second, registered(svc, inv, "llmgw-node01")) {
|
||||||
|
t.Fatal("an agent with no proven identity reached the control plane")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlaintextAgentIsRejectedWhenTLSIsConfigured(t *testing.T) {
|
||||||
|
// A misconfiguration worth catching at startup rather than at runtime: an
|
||||||
|
// agent holding certificates but dialling http:// would connect in the
|
||||||
|
// clear and never learn that it had.
|
||||||
|
_, clientTLS := pki(t, "llmgw-node01")
|
||||||
|
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.NodeID = "llmgw-node01"
|
||||||
|
cfg.Server = "http://127.0.0.1:9443"
|
||||||
|
cfg.TLS = clientTLS
|
||||||
|
|
||||||
|
if _, err := New(cfg, nil, slog.New(slog.DiscardHandler)); err == nil {
|
||||||
|
t.Fatal("an agent with certificates was allowed to dial a plaintext address")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBareAddressGainsTheRightScheme(t *testing.T) {
|
||||||
|
_, clientTLS := pki(t, "llmgw-node01")
|
||||||
|
|
||||||
|
secure := Config{NodeID: "n", Server: "10.0.0.140:9443", TLS: clientTLS}
|
||||||
|
if err := secure.normalize(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if secure.Server != "https://10.0.0.140:9443" {
|
||||||
|
t.Errorf("a configured agent dials %q", secure.Server)
|
||||||
|
}
|
||||||
|
|
||||||
|
plain := Config{NodeID: "n", Server: "10.0.0.140:9443"}
|
||||||
|
if err := plain.normalize(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if plain.Server != "http://10.0.0.140:9443" {
|
||||||
|
t.Errorf("an unconfigured agent dials %q", plain.Server)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRefusedAgentLearnsWhy(t *testing.T) {
|
||||||
|
// An agent that cannot register must be able to say why. Reporting a
|
||||||
|
// refusal as a clean disconnect leaves an operator with a silent reconnect
|
||||||
|
// loop and nothing to act on — and resets the backoff, so the loop is a
|
||||||
|
// fast one.
|
||||||
|
serverTLS, clientTLS := pki(t, "llmgw-node01")
|
||||||
|
_, _, url := startTLS(t, serverTLS)
|
||||||
|
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.NodeID = "llmgw-node02"
|
||||||
|
cfg.Server = url
|
||||||
|
cfg.TLS = clientTLS
|
||||||
|
cfg.Discovery.SkipGPUs = true
|
||||||
|
cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent")
|
||||||
|
cfg.Superv = supervisor.Config{WorkDir: t.TempDir(), LogLines: 8}
|
||||||
|
|
||||||
|
client, err := Client(cfg.TLS, 5*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ag, err := New(cfg, client, slog.New(slog.DiscardHandler))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = ag.session(context.Background())
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a refused session reported success")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "llmgw-node01") || !strings.Contains(err.Error(), "llmgw-node02") {
|
||||||
|
t.Errorf("the error names neither certificate nor claim: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAnAgentWithNothingRunningStillTellsTheControlPlane covers the gap that
|
||||||
|
// made a reconciler fix dead code in production.
|
||||||
|
//
|
||||||
|
// "I am running nothing" is information: it is how the control plane learns
|
||||||
|
// that an engine died while its agent survived. The service used to return
|
||||||
|
// early on an empty list, so that message never reached the reconciler — while
|
||||||
|
// the reconciler's own tests, which call it directly, kept passing.
|
||||||
|
//
|
||||||
|
// This test goes through the real stream for that reason.
|
||||||
|
func TestAnAgentWithNothingRunningStillTellsTheControlPlane(t *testing.T) {
|
||||||
|
serverTLS, clientTLS := pki(t, "llmgw-node01")
|
||||||
|
|
||||||
|
inv := inventory.NewStore()
|
||||||
|
rec := newRecorder()
|
||||||
|
svc := agentsvc.New(inv, rec, true, slog.New(slog.DiscardHandler))
|
||||||
|
path, handler := svc.Handler()
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
mux.Handle(path, handler)
|
||||||
|
|
||||||
|
tc, err := serverTLS.ServerConfig()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
raw, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
srv := httptest.NewUnstartedServer(mux)
|
||||||
|
srv.Listener = raw
|
||||||
|
srv.TLS = tc
|
||||||
|
srv.StartTLS()
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.NodeID = "llmgw-node01"
|
||||||
|
cfg.Server = "https://" + raw.Addr().String()
|
||||||
|
cfg.TLS = clientTLS
|
||||||
|
runAgent(t, cfg)
|
||||||
|
|
||||||
|
// The agent has no shards at all, which is the whole point.
|
||||||
|
if !eventually(t, 10*time.Second, func() bool {
|
||||||
|
return len(rec.adoptCalls()) > 0
|
||||||
|
}) {
|
||||||
|
t.Fatal("an agent running nothing never reached the reconciler")
|
||||||
|
}
|
||||||
|
for _, call := range rec.adoptCalls() {
|
||||||
|
if call.nodeID != "llmgw-node01" {
|
||||||
|
t.Errorf("adopt was told node %q", call.nodeID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
||||||
|
)
|
||||||
|
|
||||||
|
// agentWithCache is an agent that only knows where its weights live, which is
|
||||||
|
// all removeWeights needs.
|
||||||
|
func agentWithCache(t *testing.T, dir string) *Agent {
|
||||||
|
t.Helper()
|
||||||
|
return &Agent{
|
||||||
|
cfg: Config{ModelDirs: []string{dir}},
|
||||||
|
log: slog.New(slog.DiscardHandler),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func write(t *testing.T, path string) string {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(path, []byte("weights"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheAgentRefusesToDeleteWhatItDidNotFetch(t *testing.T) {
|
||||||
|
// The control plane computes these paths, but the node holds the files. A
|
||||||
|
// control plane that is wrong, newer than this agent, or tampered with must
|
||||||
|
// not be able to talk it into deleting a model an operator spent a week
|
||||||
|
// copying here. So the agent checks again, on its own.
|
||||||
|
dir := t.TempDir()
|
||||||
|
a := agentWithCache(t, dir)
|
||||||
|
|
||||||
|
operators := write(t, filepath.Join(dir, "qwen2.5-coder-14b-instruct-q8_0.gguf"))
|
||||||
|
fetched := write(t, weights.CachePathFor(dir, "https://models.internal/gone.gguf"))
|
||||||
|
|
||||||
|
a.removeWeights([]string{operators, fetched})
|
||||||
|
|
||||||
|
if _, err := os.Stat(operators); err != nil {
|
||||||
|
t.Errorf("a file llmgw never fetched was deleted: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(fetched); !os.IsNotExist(err) {
|
||||||
|
t.Errorf("the fetched file was not removed: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheAgentRefusesPathsOutsideItsCache(t *testing.T) {
|
||||||
|
// Even with a name that looks like one of ours. The cache directory is the
|
||||||
|
// boundary, and traversal out of it is the attack this closes.
|
||||||
|
dir := t.TempDir()
|
||||||
|
elsewhere := t.TempDir()
|
||||||
|
a := agentWithCache(t, dir)
|
||||||
|
|
||||||
|
// A correctly named cache file, in the wrong directory.
|
||||||
|
outside := write(t, weights.CachePathFor(elsewhere, "https://models.internal/x.gguf"))
|
||||||
|
// And the same thing reached by traversal from inside the cache.
|
||||||
|
traversal := filepath.Join(dir, "..", filepath.Base(elsewhere),
|
||||||
|
filepath.Base(outside))
|
||||||
|
|
||||||
|
a.removeWeights([]string{outside, traversal})
|
||||||
|
|
||||||
|
if _, err := os.Stat(outside); err != nil {
|
||||||
|
t.Errorf("a file outside the cache was deleted: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRemovingAFileThatIsAlreadyGoneIsNotAFailure(t *testing.T) {
|
||||||
|
// The expected outcome of a retry. It must not stop the rest of the batch.
|
||||||
|
dir := t.TempDir()
|
||||||
|
a := agentWithCache(t, dir)
|
||||||
|
|
||||||
|
missing := weights.CachePathFor(dir, "https://models.internal/never.gguf")
|
||||||
|
present := write(t, weights.CachePathFor(dir, "https://models.internal/here.gguf"))
|
||||||
|
|
||||||
|
a.removeWeights([]string{missing, present})
|
||||||
|
|
||||||
|
if _, err := os.Stat(present); !os.IsNotExist(err) {
|
||||||
|
t.Errorf("a missing file stopped the batch: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWithNoCacheConfiguredNothingIsDeleted(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
a := &Agent{cfg: Config{}, log: slog.New(slog.DiscardHandler)}
|
||||||
|
kept := write(t, weights.CachePathFor(dir, "https://models.internal/x.gguf"))
|
||||||
|
|
||||||
|
a.removeWeights([]string{kept})
|
||||||
|
|
||||||
|
if _, err := os.Stat(kept); err != nil {
|
||||||
|
t.Errorf("an agent with no cache directory deleted something: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestAgentStopsWhenContextEnds pins down something cancellation does not do by
|
||||||
|
// itself. A bidirectional stream spends its life blocked reading, and HTTP/2
|
||||||
|
// stops watching the context once the request headers are away, so an agent
|
||||||
|
// that only cancels keeps reading until something kills it. Under systemd that
|
||||||
|
// is the full stop timeout on every restart, followed by SIGKILL.
|
||||||
|
func TestAgentStopsWhenContextEnds(t *testing.T) {
|
||||||
|
serverTLS, clientTLS := pki(t, "llmgw-node01")
|
||||||
|
svc, inv, url := startTLS(t, serverTLS)
|
||||||
|
|
||||||
|
cfg := DefaultConfig()
|
||||||
|
cfg.NodeID = "llmgw-node01"
|
||||||
|
cfg.Server = url
|
||||||
|
cfg.TLS = clientTLS
|
||||||
|
cfg.HeartbeatInterval = 50 * time.Millisecond
|
||||||
|
cfg.Discovery.SkipGPUs = true
|
||||||
|
cfg.Discovery.IBRoot = filepath.Join(t.TempDir(), "absent")
|
||||||
|
cfg.Superv = supervisor.Config{
|
||||||
|
WorkDir: t.TempDir(), LogLines: 8,
|
||||||
|
StopTimeout: time.Second, HealthInterval: 20 * time.Millisecond,
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err := Client(cfg.TLS, 5*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ag, err := New(cfg, client, slog.New(slog.DiscardHandler))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { _ = ag.Run(ctx); close(done) }()
|
||||||
|
|
||||||
|
// Cancelling before the stream is established would prove nothing: the
|
||||||
|
// blocked read is the case that matters.
|
||||||
|
if !eventually(t, 10*time.Second, registered(svc, inv, "llmgw-node01")) {
|
||||||
|
t.Fatal("the agent never registered")
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the agent did not stop within 5s of being told to")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
package supervisor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HTTPHealth checks an engine over HTTP.
|
||||||
|
//
|
||||||
|
// Readiness is two questions, not one (plan §5.4): /health says the process is
|
||||||
|
// up, and /v1/models says the weights finished loading and the model is being
|
||||||
|
// served. An engine answers /health long before it can serve a request, so
|
||||||
|
// routing on /health alone sends traffic into a model that is still loading.
|
||||||
|
type HTTPHealth struct {
|
||||||
|
client *http.Client
|
||||||
|
healthPath string
|
||||||
|
modelsPath string
|
||||||
|
requireName bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// HTTPHealthOption customizes the checker.
|
||||||
|
type HTTPHealthOption func(*HTTPHealth)
|
||||||
|
|
||||||
|
// WithHealthPath overrides the readiness path.
|
||||||
|
func WithHealthPath(p string) HTTPHealthOption {
|
||||||
|
return func(h *HTTPHealth) { h.healthPath = p }
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithoutModelCheck skips the /v1/models step, for engines that do not expose it.
|
||||||
|
func WithoutModelCheck() HTTPHealthOption {
|
||||||
|
return func(h *HTTPHealth) { h.requireName = false }
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewHTTPHealth builds a checker.
|
||||||
|
func NewHTTPHealth(timeout time.Duration, opts ...HTTPHealthOption) *HTTPHealth {
|
||||||
|
if timeout <= 0 {
|
||||||
|
timeout = 3 * time.Second
|
||||||
|
}
|
||||||
|
h := &HTTPHealth{
|
||||||
|
client: &http.Client{Timeout: timeout},
|
||||||
|
healthPath: "/health",
|
||||||
|
modelsPath: "/v1/models",
|
||||||
|
requireName: true,
|
||||||
|
}
|
||||||
|
for _, o := range opts {
|
||||||
|
o(h)
|
||||||
|
}
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ready reports whether the engine at baseURL is serving model.
|
||||||
|
func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
|
||||||
|
if err := h.get(ctx, baseURL+h.healthPath, nil); err != nil {
|
||||||
|
return fmt.Errorf("health: %w", err)
|
||||||
|
}
|
||||||
|
if !h.requireName || model == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var list struct {
|
||||||
|
Data []struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
} `json:"data"`
|
||||||
|
}
|
||||||
|
if err := h.get(ctx, baseURL+h.modelsPath, &list); err != nil {
|
||||||
|
return fmt.Errorf("models: %w", err)
|
||||||
|
}
|
||||||
|
for _, d := range list.Data {
|
||||||
|
if d.ID == model {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Errorf("engine is up but does not serve %q yet", model)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *HTTPHealth) get(ctx context.Context, url string, out any) error {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
resp, err := h.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<10))
|
||||||
|
return fmt.Errorf("status %s", resp.Status)
|
||||||
|
}
|
||||||
|
if out == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return json.NewDecoder(resp.Body).Decode(out)
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
package supervisor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// engineStub serves configurable /health and /v1/models responses.
|
||||||
|
func engineStub(t *testing.T, healthy bool, models ...string) *httptest.Server {
|
||||||
|
t.Helper()
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
if !healthy {
|
||||||
|
http.Error(w, "loading", http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
})
|
||||||
|
mux.HandleFunc("GET /v1/models", func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString(`{"object":"list","data":[`)
|
||||||
|
for i, m := range models {
|
||||||
|
if i > 0 {
|
||||||
|
b.WriteString(",")
|
||||||
|
}
|
||||||
|
b.WriteString(`{"id":"` + m + `"}`)
|
||||||
|
}
|
||||||
|
b.WriteString(`]}`)
|
||||||
|
io.WriteString(w, b.String())
|
||||||
|
})
|
||||||
|
srv := httptest.NewServer(mux)
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadyWhenServingModel(t *testing.T) {
|
||||||
|
srv := engineStub(t, true, "llama-3.3-70b")
|
||||||
|
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b"); err != nil {
|
||||||
|
t.Errorf("a serving engine was reported unready: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNotReadyWhileLoading(t *testing.T) {
|
||||||
|
// The case that matters: the process is up but the weights are not loaded.
|
||||||
|
// Routing on /health alone would send traffic into a loading model.
|
||||||
|
srv := engineStub(t, false, "llama-3.3-70b")
|
||||||
|
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("an unhealthy engine was reported ready")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "health") {
|
||||||
|
t.Errorf("err=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNotReadyWhenModelMissing(t *testing.T) {
|
||||||
|
// /health passes but the model is not listed: the engine is up and serving
|
||||||
|
// something else, or still registering.
|
||||||
|
srv := engineStub(t, true, "some-other-model")
|
||||||
|
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a model that is not served was reported ready")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "does not serve") {
|
||||||
|
t.Errorf("err=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestModelCheckCanBeSkipped(t *testing.T) {
|
||||||
|
srv := engineStub(t, true)
|
||||||
|
h := NewHTTPHealth(time.Second, WithoutModelCheck())
|
||||||
|
if err := h.Ready(t.Context(), srv.URL, "anything"); err != nil {
|
||||||
|
t.Errorf("the model check was not skipped: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnreachableEngine(t *testing.T) {
|
||||||
|
h := NewHTTPHealth(200 * time.Millisecond)
|
||||||
|
if err := h.Ready(t.Context(), "http://127.0.0.1:1", "m"); err == nil {
|
||||||
|
t.Error("an unreachable engine was reported ready")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
package supervisor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// killGroup signals a whole process group.
|
||||||
|
//
|
||||||
|
// Engines fork worker processes that hold GPU memory. Signalling only the
|
||||||
|
// leader leaves those workers alive, and the next launch then fails to
|
||||||
|
// allocate — with an error that points at the new process rather than the old.
|
||||||
|
func killGroup(pid int, sig syscall.Signal) error {
|
||||||
|
// The negative pid addresses the group. Setpgid at launch made the leader's
|
||||||
|
// pid the group id.
|
||||||
|
if err := syscall.Kill(-pid, sig); err != nil {
|
||||||
|
if err == syscall.ESRCH {
|
||||||
|
return os.ErrProcessDone
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// processAlive reports whether a pid belongs to a process that is still doing
|
||||||
|
// something.
|
||||||
|
//
|
||||||
|
// A zombie is not. It has exited and is only waiting to be reaped, but
|
||||||
|
// kill(pid, 0) still succeeds on it — so a naive check reports a dead engine as
|
||||||
|
// running forever. That happens for real whenever an engine's parent dies
|
||||||
|
// before it does and nothing reaps the orphan: in a container, PID 1 is usually
|
||||||
|
// the workload rather than an init, and orphans simply accumulate.
|
||||||
|
func processAlive(pid int) bool {
|
||||||
|
if pid <= 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// Signal 0 performs the permission and existence checks without delivering
|
||||||
|
// anything.
|
||||||
|
if err := syscall.Kill(pid, 0); err != nil {
|
||||||
|
// EPERM means it exists but belongs to someone else, which for our
|
||||||
|
// purposes is still "there".
|
||||||
|
return err == syscall.EPERM
|
||||||
|
}
|
||||||
|
return !isZombie(pid)
|
||||||
|
}
|
||||||
|
|
||||||
|
// isZombie reports whether a pid has exited but not been reaped.
|
||||||
|
//
|
||||||
|
// Only Linux exposes this; elsewhere there is no procfs and the answer is
|
||||||
|
// false, which degrades to the old behaviour rather than to something worse.
|
||||||
|
func isZombie(pid int) bool {
|
||||||
|
b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat")
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// The comm field is parenthesised and may itself contain spaces or
|
||||||
|
// parentheses, so the state character is found from the last ')'.
|
||||||
|
i := bytes.LastIndexByte(b, ')')
|
||||||
|
if i < 0 || i+2 >= len(b) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return b[i+2] == 'Z'
|
||||||
|
}
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
package supervisor
|
||||||
|
|
||||||
|
import "sync"
|
||||||
|
|
||||||
|
// ringBuffer keeps the last n lines of output.
|
||||||
|
//
|
||||||
|
// Engine logs are the only diagnosis available when a shard fails to load, but
|
||||||
|
// they are also unbounded — a model loading for ten minutes can produce
|
||||||
|
// megabytes. Keeping a bounded tail makes `ctl logs` useful without letting a
|
||||||
|
// chatty engine consume the agent's memory.
|
||||||
|
type ringBuffer struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
lines []string
|
||||||
|
next int
|
||||||
|
full bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRingBuffer(n int) *ringBuffer {
|
||||||
|
if n <= 0 {
|
||||||
|
n = 256
|
||||||
|
}
|
||||||
|
return &ringBuffer{lines: make([]string, n)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *ringBuffer) add(line string) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
r.lines[r.next] = line
|
||||||
|
r.next = (r.next + 1) % len(r.lines)
|
||||||
|
if r.next == 0 {
|
||||||
|
r.full = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tail returns up to n most recent lines, oldest first.
|
||||||
|
func (r *ringBuffer) tail(n int) []string {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
|
||||||
|
size := r.next
|
||||||
|
if r.full {
|
||||||
|
size = len(r.lines)
|
||||||
|
}
|
||||||
|
if n <= 0 || n > size {
|
||||||
|
n = size
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]string, 0, n)
|
||||||
|
for i := size - n; i < size; i++ {
|
||||||
|
idx := i
|
||||||
|
if r.full {
|
||||||
|
idx = (r.next + i) % len(r.lines)
|
||||||
|
}
|
||||||
|
out = append(out, r.lines[idx])
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
// Package supervisor runs and watches engine processes (plan §5.4).
|
||||||
|
package supervisor
|
||||||
|
|
||||||
|
import "fmt"
|
||||||
|
|
||||||
|
// State is where a shard is in its lifecycle.
|
||||||
|
//
|
||||||
|
// Loading is separate from Starting because model loading dominates startup:
|
||||||
|
// a 70B model takes minutes, and an operator watching a shard sit in one state
|
||||||
|
// for ten minutes needs to know whether that is normal.
|
||||||
|
type State int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// Pending means the shard is known but not started.
|
||||||
|
Pending State = iota
|
||||||
|
// Starting means the process has been launched but has not opened its port.
|
||||||
|
Starting
|
||||||
|
// Loading means the port is open but the model is not ready.
|
||||||
|
Loading
|
||||||
|
// Ready means health checks pass and the model is served.
|
||||||
|
Ready
|
||||||
|
// Draining means no new work; existing requests finish.
|
||||||
|
Draining
|
||||||
|
// Stopped means the process exited as intended.
|
||||||
|
Stopped
|
||||||
|
// Failed means the process exited unexpectedly or never became ready.
|
||||||
|
Failed
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s State) String() string {
|
||||||
|
switch s {
|
||||||
|
case Pending:
|
||||||
|
return "pending"
|
||||||
|
case Starting:
|
||||||
|
return "starting"
|
||||||
|
case Loading:
|
||||||
|
return "loading"
|
||||||
|
case Ready:
|
||||||
|
return "ready"
|
||||||
|
case Draining:
|
||||||
|
return "draining"
|
||||||
|
case Stopped:
|
||||||
|
return "stopped"
|
||||||
|
case Failed:
|
||||||
|
return "failed"
|
||||||
|
default:
|
||||||
|
return fmt.Sprintf("unknown(%d)", int(s))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Terminal reports whether the shard has finished for good.
|
||||||
|
func (s State) Terminal() bool { return s == Stopped || s == Failed }
|
||||||
|
|
||||||
|
// Live reports whether a process should be running in this state.
|
||||||
|
func (s State) Live() bool {
|
||||||
|
return s == Starting || s == Loading || s == Ready || s == Draining
|
||||||
|
}
|
||||||
@@ -0,0 +1,771 @@
|
|||||||
|
package supervisor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Config configures the supervisor.
|
||||||
|
type Config struct {
|
||||||
|
// WorkDir holds pid files and per-shard scratch space.
|
||||||
|
WorkDir string `koanf:"work_dir"`
|
||||||
|
// LogLines is how many lines of engine output to retain per shard.
|
||||||
|
LogLines int `koanf:"log_lines"`
|
||||||
|
// StopTimeout is how long a process gets to exit after SIGTERM before it is
|
||||||
|
// killed. Engines need time to free GPU memory cleanly.
|
||||||
|
StopTimeout time.Duration `koanf:"stop_timeout"`
|
||||||
|
// HealthInterval is how often a starting shard is polled.
|
||||||
|
HealthInterval time.Duration `koanf:"health_interval"`
|
||||||
|
// RendezvousTimeout bounds the wait for a multi-node rendezvous point to
|
||||||
|
// start listening.
|
||||||
|
RendezvousTimeout time.Duration `koanf:"rendezvous_timeout"`
|
||||||
|
// AliveGrace is how long a non-serving rank must stay up before it counts
|
||||||
|
// as ready. There is no external signal for a worker, so staying alive is
|
||||||
|
// the only evidence available.
|
||||||
|
AliveGrace time.Duration `koanf:"alive_grace"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultConfig returns sensible defaults.
|
||||||
|
func DefaultConfig() Config {
|
||||||
|
return Config{
|
||||||
|
WorkDir: "/var/lib/llmgw-agent",
|
||||||
|
LogLines: 512,
|
||||||
|
StopTimeout: 60 * time.Second,
|
||||||
|
HealthInterval: 2 * time.Second,
|
||||||
|
RendezvousTimeout: 5 * time.Minute,
|
||||||
|
AliveGrace: 5 * time.Second,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) normalize() {
|
||||||
|
d := DefaultConfig()
|
||||||
|
if c.WorkDir == "" {
|
||||||
|
c.WorkDir = d.WorkDir
|
||||||
|
}
|
||||||
|
if c.LogLines <= 0 {
|
||||||
|
c.LogLines = d.LogLines
|
||||||
|
}
|
||||||
|
if c.StopTimeout <= 0 {
|
||||||
|
c.StopTimeout = d.StopTimeout
|
||||||
|
}
|
||||||
|
if c.HealthInterval <= 0 {
|
||||||
|
c.HealthInterval = d.HealthInterval
|
||||||
|
}
|
||||||
|
if c.RendezvousTimeout <= 0 {
|
||||||
|
c.RendezvousTimeout = d.RendezvousTimeout
|
||||||
|
}
|
||||||
|
if c.AliveGrace <= 0 {
|
||||||
|
c.AliveGrace = d.AliveGrace
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// HealthChecker reports whether a shard is serving. It is an interface so the
|
||||||
|
// supervisor can be tested without an HTTP engine.
|
||||||
|
type HealthChecker interface {
|
||||||
|
// Ready reports whether the shard at baseURL serves model.
|
||||||
|
Ready(ctx context.Context, baseURL, model string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Shard is one supervised engine process.
|
||||||
|
type Shard struct {
|
||||||
|
ID string
|
||||||
|
Spec engines.ShardSpec
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
state State
|
||||||
|
pid int
|
||||||
|
err error
|
||||||
|
started time.Time
|
||||||
|
readyAt time.Time
|
||||||
|
exitCode int
|
||||||
|
|
||||||
|
logs *ringBuffer
|
||||||
|
cmd *exec.Cmd
|
||||||
|
// sidecars are the shard's supporting processes, torn down with it.
|
||||||
|
sidecars []*exec.Cmd
|
||||||
|
cancel context.CancelFunc
|
||||||
|
done chan struct{}
|
||||||
|
// adopted marks a shard inherited from a previous agent: it has no
|
||||||
|
// exec.Cmd, so it is watched by polling and killed by signal alone.
|
||||||
|
adopted bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status is a snapshot of a shard, safe to send to the control plane.
|
||||||
|
type Status struct {
|
||||||
|
ID string
|
||||||
|
State State
|
||||||
|
PID int
|
||||||
|
Err string
|
||||||
|
Started time.Time
|
||||||
|
ReadyAt time.Time
|
||||||
|
ExitCode int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status returns the shard's current status.
|
||||||
|
func (s *Shard) Status() Status {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
st := Status{
|
||||||
|
ID: s.ID, State: s.state, PID: s.pid,
|
||||||
|
Started: s.started, ReadyAt: s.readyAt, ExitCode: s.exitCode,
|
||||||
|
}
|
||||||
|
if s.err != nil {
|
||||||
|
st.Err = s.err.Error()
|
||||||
|
}
|
||||||
|
return st
|
||||||
|
}
|
||||||
|
|
||||||
|
// Logs returns the last n lines of engine output.
|
||||||
|
func (s *Shard) Logs(n int) []string { return s.logs.tail(n) }
|
||||||
|
|
||||||
|
func (s *Shard) setState(st State, err error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
s.state = st
|
||||||
|
if err != nil {
|
||||||
|
s.err = err
|
||||||
|
}
|
||||||
|
if st == Ready && s.readyAt.IsZero() {
|
||||||
|
s.readyAt = time.Now()
|
||||||
|
}
|
||||||
|
s.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Supervisor starts, watches and stops engine processes on this node.
|
||||||
|
type Supervisor struct {
|
||||||
|
cfg Config
|
||||||
|
log *slog.Logger
|
||||||
|
health HealthChecker
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
shards map[string]*Shard
|
||||||
|
}
|
||||||
|
|
||||||
|
// New builds a supervisor.
|
||||||
|
func New(cfg Config, health HealthChecker, log *slog.Logger) (*Supervisor, error) {
|
||||||
|
cfg.normalize()
|
||||||
|
if log == nil {
|
||||||
|
log = slog.Default()
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(cfg.WorkDir, 0o750); err != nil {
|
||||||
|
return nil, fmt.Errorf("work dir: %w", err)
|
||||||
|
}
|
||||||
|
return &Supervisor{cfg: cfg, log: log, health: health, shards: map[string]*Shard{}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrAlreadyRunning means a shard with that ID is already supervised.
|
||||||
|
var ErrAlreadyRunning = errors.New("shard is already running")
|
||||||
|
|
||||||
|
// Start launches a shard.
|
||||||
|
//
|
||||||
|
// It is idempotent by ID: a repeated StartShard for the same shard is a no-op
|
||||||
|
// rather than a second process, because the control plane retries commands
|
||||||
|
// (plan §5.6).
|
||||||
|
func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmds []engines.Command) (*Shard, error) {
|
||||||
|
if len(cmds) == 0 {
|
||||||
|
return nil, fmt.Errorf("shard %s: no commands to run", id)
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
if existing, ok := s.shards[id]; ok {
|
||||||
|
s.mu.Unlock()
|
||||||
|
if existing.Status().State.Terminal() {
|
||||||
|
s.Remove(id)
|
||||||
|
} else {
|
||||||
|
return existing, ErrAlreadyRunning
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
}
|
||||||
|
|
||||||
|
sh := &Shard{
|
||||||
|
ID: id,
|
||||||
|
Spec: spec,
|
||||||
|
state: Pending,
|
||||||
|
logs: newRingBuffer(s.cfg.LogLines),
|
||||||
|
started: time.Now(),
|
||||||
|
done: make(chan struct{}),
|
||||||
|
}
|
||||||
|
s.shards[id] = sh
|
||||||
|
s.mu.Unlock()
|
||||||
|
|
||||||
|
if err := s.launch(ctx, sh, cmds); err != nil {
|
||||||
|
sh.setState(Failed, err)
|
||||||
|
close(sh.done)
|
||||||
|
return sh, err
|
||||||
|
}
|
||||||
|
return sh, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// launch starts a shard's processes in order.
|
||||||
|
//
|
||||||
|
// Every command but the last is a sidecar: it keeps running for the shard's
|
||||||
|
// lifetime and is torn down with it, but its exit is not what defines the
|
||||||
|
// shard's. The last command is the engine, and its exit is the shard's exit.
|
||||||
|
func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Command) error {
|
||||||
|
runCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))
|
||||||
|
sh.cancel = cancel
|
||||||
|
|
||||||
|
for i, c := range cmds[:len(cmds)-1] {
|
||||||
|
proc, err := s.spawn(runCtx, sh, c)
|
||||||
|
if err != nil {
|
||||||
|
cancel()
|
||||||
|
return fmt.Errorf("sidecar %d (%s): %w", i, c.Name, err)
|
||||||
|
}
|
||||||
|
sh.mu.Lock()
|
||||||
|
sh.sidecars = append(sh.sidecars, proc)
|
||||||
|
sh.mu.Unlock()
|
||||||
|
|
||||||
|
if c.WaitForLeaderPort {
|
||||||
|
// The next process cannot start until the rendezvous point is
|
||||||
|
// listening: a Ray worker cannot join a head that is not up, and
|
||||||
|
// the leader's server cannot claim GPUs from a cluster that does
|
||||||
|
// not exist yet.
|
||||||
|
addr := fmt.Sprintf("%s:%d", sh.Spec.LeaderAddr, sh.Spec.LeaderPort)
|
||||||
|
if err := waitForPort(runCtx, addr, s.cfg.RendezvousTimeout); err != nil {
|
||||||
|
cancel()
|
||||||
|
return fmt.Errorf("waiting for %s (%s): %w", addr, c.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := cmds[len(cmds)-1]
|
||||||
|
|
||||||
|
c := exec.CommandContext(runCtx, cmd.Path, cmd.Args...)
|
||||||
|
c.Env = cmd.Env
|
||||||
|
c.Dir = s.cfg.WorkDir
|
||||||
|
// Its own process group, so stopping kills the engine's children too.
|
||||||
|
// Engines spawn workers; signalling only the parent leaves those workers
|
||||||
|
// holding GPU memory.
|
||||||
|
c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||||
|
// CommandContext kills only the leader, so replace that with a group kill.
|
||||||
|
c.Cancel = func() error { return killGroup(c.Process.Pid, syscall.SIGKILL) }
|
||||||
|
|
||||||
|
stdout, err := c.StdoutPipe()
|
||||||
|
if err != nil {
|
||||||
|
cancel()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c.Stderr = c.Stdout.(io.Writer) // interleave, as the engine writes them
|
||||||
|
|
||||||
|
if err := c.Start(); err != nil {
|
||||||
|
cancel()
|
||||||
|
return fmt.Errorf("starting %s: %w", cmd.Path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sh.mu.Lock()
|
||||||
|
sh.cmd = c
|
||||||
|
sh.pid = c.Process.Pid
|
||||||
|
sh.state = Starting
|
||||||
|
sh.mu.Unlock()
|
||||||
|
|
||||||
|
if err := s.writeState(sh); err != nil {
|
||||||
|
s.log.Warn("could not write pid file; adoption after a restart will not work",
|
||||||
|
"shard", sh.ID, "err", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
go s.pump(sh, stdout, cmd.Name)
|
||||||
|
go s.wait(sh)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// spawn starts one process in its own group and pipes its output to the log.
|
||||||
|
func (s *Supervisor) spawn(ctx context.Context, sh *Shard, c engines.Command) (*exec.Cmd, error) {
|
||||||
|
proc := exec.CommandContext(ctx, c.Path, c.Args...)
|
||||||
|
proc.Env = c.Env
|
||||||
|
proc.Dir = s.cfg.WorkDir
|
||||||
|
proc.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||||
|
proc.Cancel = func() error { return killGroup(proc.Process.Pid, syscall.SIGKILL) }
|
||||||
|
|
||||||
|
stdout, err := proc.StdoutPipe()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
proc.Stderr = proc.Stdout.(io.Writer)
|
||||||
|
|
||||||
|
if err := proc.Start(); err != nil {
|
||||||
|
return nil, fmt.Errorf("starting %s: %w", c.Path, err)
|
||||||
|
}
|
||||||
|
go s.pump(sh, stdout, c.Name)
|
||||||
|
// Reaped so it does not become a zombie; its exit does not end the shard.
|
||||||
|
go func() { _ = proc.Wait() }()
|
||||||
|
return proc, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitForPort blocks until addr accepts a connection.
|
||||||
|
func waitForPort(ctx context.Context, addr string, timeout time.Duration) error {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
var d net.Dialer
|
||||||
|
for {
|
||||||
|
conn, err := d.DialContext(ctx, "tcp", addr)
|
||||||
|
if err == nil {
|
||||||
|
_ = conn.Close()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return fmt.Errorf("not listening after %v: %w", timeout, ctx.Err())
|
||||||
|
case <-time.After(200 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pump copies a process's output into the ring buffer and the log.
|
||||||
|
func (s *Supervisor) pump(sh *Shard, r io.Reader, name string) {
|
||||||
|
sc := bufio.NewScanner(r)
|
||||||
|
// Engine tracebacks and progress bars produce long lines; the default
|
||||||
|
// 64 KiB limit truncates them exactly when they matter.
|
||||||
|
sc.Buffer(make([]byte, 0, 64<<10), 1<<20)
|
||||||
|
|
||||||
|
for sc.Scan() {
|
||||||
|
line := sc.Text()
|
||||||
|
if name != "" {
|
||||||
|
// Prefixed so the two processes of a Ray shard stay distinguishable
|
||||||
|
// in a single log tail.
|
||||||
|
line = "[" + name + "] " + line
|
||||||
|
}
|
||||||
|
sh.logs.add(line)
|
||||||
|
s.log.Debug("engine", "shard", sh.ID, "line", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// wait reaps the process and records why it ended.
|
||||||
|
func (s *Supervisor) wait(sh *Shard) {
|
||||||
|
defer close(sh.done)
|
||||||
|
|
||||||
|
err := sh.cmd.Wait()
|
||||||
|
|
||||||
|
sh.mu.Lock()
|
||||||
|
prev := sh.state
|
||||||
|
if ee := new(exec.ExitError); errors.As(err, &ee) {
|
||||||
|
sh.exitCode = ee.ExitCode()
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case prev == Draining || prev == Stopped:
|
||||||
|
// Expected: we asked it to stop.
|
||||||
|
sh.state = Stopped
|
||||||
|
case err == nil:
|
||||||
|
// An engine exiting on its own is a failure, not a success: it is
|
||||||
|
// supposed to serve until told otherwise.
|
||||||
|
sh.state = Failed
|
||||||
|
sh.err = errors.New("engine exited unexpectedly")
|
||||||
|
default:
|
||||||
|
sh.state = Failed
|
||||||
|
sh.err = err
|
||||||
|
}
|
||||||
|
final, code := sh.state, sh.exitCode
|
||||||
|
sh.mu.Unlock()
|
||||||
|
|
||||||
|
s.removeState(sh)
|
||||||
|
if final == Failed {
|
||||||
|
s.log.Error("shard failed", "shard", sh.ID, "exit_code", code, "err", err,
|
||||||
|
"last_output", strings.Join(sh.Logs(5), " | "))
|
||||||
|
} else {
|
||||||
|
s.log.Info("shard stopped", "shard", sh.ID, "exit_code", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WaitReady polls until the shard serves, it fails, or the timeout expires.
|
||||||
|
//
|
||||||
|
// The timeout comes from the profile because it is model-dependent: loading a
|
||||||
|
// 70B model across two nodes legitimately takes tens of minutes, while a small
|
||||||
|
// model failing to load for ten minutes means something is wrong.
|
||||||
|
func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model string, timeout time.Duration) error {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
t := time.NewTicker(s.cfg.HealthInterval)
|
||||||
|
defer t.Stop()
|
||||||
|
|
||||||
|
sh.setState(Loading, nil)
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-sh.done:
|
||||||
|
return fmt.Errorf("shard %s exited during startup: %s", sh.ID, sh.Status().Err)
|
||||||
|
case <-ctx.Done():
|
||||||
|
last := strings.Join(sh.Logs(10), "\n")
|
||||||
|
return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last)
|
||||||
|
case <-t.C:
|
||||||
|
if err := s.health.Ready(ctx, baseURL, model); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sh.setState(Ready, nil)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WaitAlive is readiness for a rank that serves no HTTP.
|
||||||
|
//
|
||||||
|
// A worker in a multi-node replica has no endpoint to poll, so the only
|
||||||
|
// evidence available is that it started and stayed started. That is weaker than
|
||||||
|
// a health check, and deliberately so: the replica's real readiness is the
|
||||||
|
// leader answering, which only happens once every worker has joined. Polling a
|
||||||
|
// worker's port instead would time out and fail a replica that is coming up
|
||||||
|
// perfectly well.
|
||||||
|
func (s *Supervisor) WaitAlive(ctx context.Context, sh *Shard) error {
|
||||||
|
sh.setState(Loading, nil)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-sh.done:
|
||||||
|
return fmt.Errorf("shard %s exited during startup: %s", sh.ID, sh.Status().Err)
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-time.After(s.cfg.AliveGrace):
|
||||||
|
}
|
||||||
|
|
||||||
|
if st := sh.Status().State; st.Terminal() {
|
||||||
|
return fmt.Errorf("shard %s is %s: %s", sh.ID, st, sh.Status().Err)
|
||||||
|
}
|
||||||
|
sh.setState(Ready, nil)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drain marks a shard as taking no new work. The router stops routing to it;
|
||||||
|
// the process keeps running until Stop.
|
||||||
|
func (s *Supervisor) Drain(id string) error {
|
||||||
|
sh, err := s.Get(id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if st := sh.Status().State; !st.Live() {
|
||||||
|
return fmt.Errorf("shard %s is %s, cannot drain", id, st)
|
||||||
|
}
|
||||||
|
sh.setState(Draining, nil)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop terminates a shard: SIGTERM to the whole group, then SIGKILL after
|
||||||
|
// StopTimeout.
|
||||||
|
func (s *Supervisor) Stop(ctx context.Context, id string) error {
|
||||||
|
sh, err := s.Get(id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
sh.mu.Lock()
|
||||||
|
pid := sh.pid
|
||||||
|
sidecars := slices.Clone(sh.sidecars)
|
||||||
|
if sh.state.Live() {
|
||||||
|
sh.state = Draining // marks the exit as expected for wait() and poll()
|
||||||
|
}
|
||||||
|
sh.mu.Unlock()
|
||||||
|
|
||||||
|
// Sidecars go too. A Ray head left behind holds the port the next replica
|
||||||
|
// needs, and would let a later shard join a cluster that serves nothing.
|
||||||
|
defer func() {
|
||||||
|
for _, sc := range sidecars {
|
||||||
|
if sc.Process == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := killGroup(sc.Process.Pid, syscall.SIGTERM); err != nil &&
|
||||||
|
!errors.Is(err, os.ErrProcessDone) {
|
||||||
|
s.log.Warn("stopping a sidecar failed", "shard", id, "pid", sc.Process.Pid, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// A pid of zero means the process never started; an adopted shard has a pid
|
||||||
|
// but no exec.Cmd, and must still be killed by signal.
|
||||||
|
if pid == 0 {
|
||||||
|
sh.setState(Stopped, nil)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Signal the group, not the process: engines fork workers that hold GPU
|
||||||
|
// memory, and those outlive a signal sent only to the leader.
|
||||||
|
if err := killGroup(pid, syscall.SIGTERM); err != nil && !errors.Is(err, os.ErrProcessDone) {
|
||||||
|
s.log.Warn("SIGTERM failed", "shard", id, "err", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// force kills the group outright. exec.Cmd's cancel does this for shards we
|
||||||
|
// started; an adopted shard has none, so signal its group directly.
|
||||||
|
force := func() {
|
||||||
|
if sh.cancel != nil {
|
||||||
|
sh.cancel()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := killGroup(pid, syscall.SIGKILL); err != nil && !errors.Is(err, os.ErrProcessDone) {
|
||||||
|
s.log.Warn("SIGKILL failed", "shard", id, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-sh.done:
|
||||||
|
return nil
|
||||||
|
case <-time.After(s.cfg.StopTimeout):
|
||||||
|
s.log.Warn("shard did not exit in time, killing", "shard", id, "timeout", s.cfg.StopTimeout)
|
||||||
|
force()
|
||||||
|
case <-ctx.Done():
|
||||||
|
force()
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-sh.done:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
return fmt.Errorf("shard %s survived SIGKILL", id)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// StopAll stops every live shard.
|
||||||
|
func (s *Supervisor) StopAll(ctx context.Context) {
|
||||||
|
for _, id := range s.IDs() {
|
||||||
|
if err := s.Stop(ctx, id); err != nil {
|
||||||
|
s.log.Warn("stopping shard failed", "shard", id, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get returns a supervised shard.
|
||||||
|
func (s *Supervisor) Get(id string) (*Shard, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
sh, ok := s.shards[id]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("unknown shard %q", id)
|
||||||
|
}
|
||||||
|
return sh, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove forgets a terminated shard.
|
||||||
|
func (s *Supervisor) Remove(id string) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
delete(s.shards, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IDs lists the supervised shards.
|
||||||
|
func (s *Supervisor) IDs() []string {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
out := make([]string, 0, len(s.shards))
|
||||||
|
for id := range s.shards {
|
||||||
|
out = append(out, id)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Statuses returns every shard's status.
|
||||||
|
func (s *Supervisor) Statuses() []Status {
|
||||||
|
s.mu.Lock()
|
||||||
|
shards := make([]*Shard, 0, len(s.shards))
|
||||||
|
for _, sh := range s.shards {
|
||||||
|
shards = append(shards, sh)
|
||||||
|
}
|
||||||
|
s.mu.Unlock()
|
||||||
|
|
||||||
|
out := make([]Status, 0, len(shards))
|
||||||
|
for _, sh := range shards {
|
||||||
|
out = append(out, sh.Status())
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- state files, for adoption after an agent restart ---
|
||||||
|
|
||||||
|
func (s *Supervisor) statePath(id string) string {
|
||||||
|
return filepath.Join(s.cfg.WorkDir, "shard-"+id+".json")
|
||||||
|
}
|
||||||
|
|
||||||
|
// shardState is what survives an agent restart.
|
||||||
|
//
|
||||||
|
// The pid alone is not enough for two separate reasons. Pids are reused, so the
|
||||||
|
// command line is kept as a fingerprint — adopting an unrelated process would
|
||||||
|
// be worse than leaking the original. And the full spec is kept because a
|
||||||
|
// recovered shard has to be described back to the control plane: without it the
|
||||||
|
// control plane cannot tell which replica the process belongs to, and would
|
||||||
|
// place a duplicate while the original keeps holding the GPUs (plan §6.5).
|
||||||
|
type shardState struct {
|
||||||
|
ShardID string `json:"shard_id"`
|
||||||
|
ReplicaID string `json:"replica_id"`
|
||||||
|
PID int `json:"pid"`
|
||||||
|
Cmdline []string `json:"cmdline"`
|
||||||
|
StartedAt time.Time `json:"started_at"`
|
||||||
|
Spec engines.ShardSpec `json:"spec"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeState records everything needed to recognise and describe the process later.
|
||||||
|
func (s *Supervisor) writeState(sh *Shard) error {
|
||||||
|
b, err := json.Marshal(shardState{
|
||||||
|
ShardID: sh.ID,
|
||||||
|
ReplicaID: sh.Spec.ReplicaID,
|
||||||
|
PID: sh.pid,
|
||||||
|
Cmdline: sh.cmd.Args,
|
||||||
|
StartedAt: sh.started,
|
||||||
|
Spec: sh.Spec,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Written atomically: a half-written state file after a crash would be
|
||||||
|
// indistinguishable from a corrupt one, and the shard would be killed.
|
||||||
|
tmp := s.statePath(sh.ID) + ".tmp"
|
||||||
|
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(tmp, s.statePath(sh.ID))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Supervisor) removeState(sh *Shard) {
|
||||||
|
_ = os.Remove(s.statePath(sh.ID))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Orphan is an engine process left behind by a previous agent.
|
||||||
|
type Orphan struct {
|
||||||
|
ShardID string
|
||||||
|
ReplicaID string
|
||||||
|
PID int
|
||||||
|
Cmdline []string
|
||||||
|
Spec engines.ShardSpec
|
||||||
|
StartedAt time.Time
|
||||||
|
// Alive reports whether the process still exists.
|
||||||
|
Alive bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindOrphans scans the work directory for processes a previous agent started.
|
||||||
|
func (s *Supervisor) FindOrphans() ([]Orphan, error) {
|
||||||
|
entries, err := os.ReadDir(s.cfg.WorkDir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var out []Orphan
|
||||||
|
for _, e := range entries {
|
||||||
|
name := e.Name()
|
||||||
|
if !strings.HasPrefix(name, "shard-") || !strings.HasSuffix(name, ".json") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := os.ReadFile(filepath.Join(s.cfg.WorkDir, name))
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var st shardState
|
||||||
|
if err := json.Unmarshal(b, &st); err != nil || st.ShardID == "" || st.PID <= 0 {
|
||||||
|
// A corrupt state file describes a process that cannot be
|
||||||
|
// identified, so it cannot be adopted safely. Leave it for
|
||||||
|
// KillOrphan to clean up by shard id from the filename.
|
||||||
|
id := strings.TrimSuffix(strings.TrimPrefix(name, "shard-"), ".json")
|
||||||
|
out = append(out, Orphan{ShardID: id})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
out = append(out, Orphan{
|
||||||
|
ShardID: st.ShardID,
|
||||||
|
ReplicaID: st.ReplicaID,
|
||||||
|
PID: st.PID,
|
||||||
|
Cmdline: st.Cmdline,
|
||||||
|
Spec: st.Spec,
|
||||||
|
StartedAt: st.StartedAt,
|
||||||
|
Alive: processAlive(st.PID),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Adopt takes over a still-running engine from a previous agent.
|
||||||
|
//
|
||||||
|
// The process cannot be waited on — it is not this process's child — so it is
|
||||||
|
// watched by polling instead. That is the whole difference between an adopted
|
||||||
|
// shard and a started one; everything downstream treats them the same.
|
||||||
|
func (s *Supervisor) Adopt(o Orphan) (*Shard, error) {
|
||||||
|
if !o.Alive {
|
||||||
|
return nil, fmt.Errorf("shard %s: process %d is gone", o.ShardID, o.PID)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
if _, exists := s.shards[o.ShardID]; exists {
|
||||||
|
s.mu.Unlock()
|
||||||
|
return nil, ErrAlreadyRunning
|
||||||
|
}
|
||||||
|
sh := &Shard{
|
||||||
|
ID: o.ShardID,
|
||||||
|
Spec: o.Spec,
|
||||||
|
state: Starting,
|
||||||
|
pid: o.PID,
|
||||||
|
logs: newRingBuffer(s.cfg.LogLines),
|
||||||
|
started: o.StartedAt,
|
||||||
|
done: make(chan struct{}),
|
||||||
|
adopted: true,
|
||||||
|
}
|
||||||
|
s.shards[o.ShardID] = sh
|
||||||
|
s.mu.Unlock()
|
||||||
|
|
||||||
|
// Output is gone with the previous agent; say so rather than presenting an
|
||||||
|
// empty log as if the engine had been silent.
|
||||||
|
sh.logs.add("[adopted from a previous agent; earlier output is not available]")
|
||||||
|
|
||||||
|
go s.poll(sh)
|
||||||
|
return sh, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// poll watches an adopted process, which cannot be reaped with Wait.
|
||||||
|
func (s *Supervisor) poll(sh *Shard) {
|
||||||
|
defer close(sh.done)
|
||||||
|
|
||||||
|
// Polled rather than waited on, so the interval bounds how quickly a stop
|
||||||
|
// is observed. Short enough that Stop does not sit out its whole timeout.
|
||||||
|
t := time.NewTicker(200 * time.Millisecond)
|
||||||
|
defer t.Stop()
|
||||||
|
for range t.C {
|
||||||
|
if processAlive(sh.pid) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sh.mu.Lock()
|
||||||
|
expected := sh.state == Draining || sh.state == Stopped
|
||||||
|
if expected {
|
||||||
|
sh.state = Stopped
|
||||||
|
} else {
|
||||||
|
sh.state = Failed
|
||||||
|
sh.err = errors.New("adopted engine exited")
|
||||||
|
}
|
||||||
|
sh.mu.Unlock()
|
||||||
|
|
||||||
|
s.removeState(sh)
|
||||||
|
s.log.Info("adopted shard exited", "shard", sh.ID, "expected", expected)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// KillOrphan terminates a leftover process group and removes its pid file.
|
||||||
|
func (s *Supervisor) KillOrphan(o Orphan) error {
|
||||||
|
if o.Alive {
|
||||||
|
if err := killGroup(o.PID, syscall.SIGTERM); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Give it a moment to free GPU memory before insisting.
|
||||||
|
deadline := time.Now().Add(s.cfg.StopTimeout)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if !processAlive(o.PID) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(200 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if processAlive(o.PID) {
|
||||||
|
_ = killGroup(o.PID, syscall.SIGKILL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// An already-missing state file is success: the previous agent may have
|
||||||
|
// reaped the process itself between the scan and this call.
|
||||||
|
if err := os.Remove(s.statePath(o.ShardID)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,788 @@
|
|||||||
|
package supervisor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||||
|
)
|
||||||
|
|
||||||
|
// scriptedHealth reports ready once the flag is set.
|
||||||
|
type scriptedHealth struct{ ready atomic.Bool }
|
||||||
|
|
||||||
|
func (h *scriptedHealth) Ready(context.Context, string, string) error {
|
||||||
|
if h.ready.Load() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return errors.New("not ready")
|
||||||
|
}
|
||||||
|
|
||||||
|
func newSupervisor(t *testing.T) (*Supervisor, *scriptedHealth) {
|
||||||
|
t.Helper()
|
||||||
|
h := &scriptedHealth{}
|
||||||
|
s, err := New(Config{
|
||||||
|
WorkDir: t.TempDir(),
|
||||||
|
LogLines: 64,
|
||||||
|
StopTimeout: 3 * time.Second,
|
||||||
|
HealthInterval: 20 * time.Millisecond,
|
||||||
|
}, h, slog.New(slog.DiscardHandler))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { s.StopAll(context.Background()) })
|
||||||
|
return s, h
|
||||||
|
}
|
||||||
|
|
||||||
|
// shellCommand builds the single-command sequence for a shell script.
|
||||||
|
func shellCommand(script string) []engines.Command {
|
||||||
|
return []engines.Command{{Path: "/bin/sh", Args: []string{"-c", script}, Env: os.Environ()}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitState(t *testing.T, sh *Shard, want State, limit time.Duration) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(limit)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if sh.Status().State == want {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatalf("shard stayed in %s, expected %s", sh.Status().State, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStartAndStop(t *testing.T) {
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
|
||||||
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if st := sh.Status(); st.State != Starting || st.PID == 0 {
|
||||||
|
t.Fatalf("status after start: %+v", st)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.Stop(t.Context(), "a"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if st := sh.Status().State; st != Stopped {
|
||||||
|
t.Errorf("state after stop: %s", st)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnexpectedExitIsFailure(t *testing.T) {
|
||||||
|
// An engine exiting on its own is a failure even with status 0: it is meant
|
||||||
|
// to serve until told otherwise.
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
|
||||||
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 0"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
waitState(t, sh, Failed, 3*time.Second)
|
||||||
|
if !strings.Contains(sh.Status().Err, "unexpectedly") {
|
||||||
|
t.Errorf("err=%q", sh.Status().Err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNonZeroExitRecordsCode(t *testing.T) {
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
|
||||||
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 42"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
waitState(t, sh, Failed, 3*time.Second)
|
||||||
|
if got := sh.Status().ExitCode; got != 42 {
|
||||||
|
t.Errorf("exit code=%d, expected 42", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStopKillsChildProcesses(t *testing.T) {
|
||||||
|
// The reason for the process group: engines fork workers that hold GPU
|
||||||
|
// memory, and signalling only the leader leaves them behind.
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
marker := filepath.Join(dir, "child.pid")
|
||||||
|
|
||||||
|
// The shell forks a child that writes its own pid, then both sleep.
|
||||||
|
script := fmt.Sprintf(`sh -c 'echo $$ > %s; sleep 30' & sleep 30`, marker)
|
||||||
|
if _, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand(script)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for the child to announce itself.
|
||||||
|
var childPID int
|
||||||
|
deadline := time.Now().Add(3 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if b, err := os.ReadFile(marker); err == nil {
|
||||||
|
if _, err := fmt.Sscanf(strings.TrimSpace(string(b)), "%d", &childPID); err == nil && childPID > 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if childPID == 0 {
|
||||||
|
t.Fatal("child never started")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.Stop(t.Context(), "a"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
deadline = time.Now().Add(3 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if !processAlive(childPID) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Errorf("child process %d survived the stop; it would still hold GPU memory", childPID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStopKillsAfterTimeout(t *testing.T) {
|
||||||
|
// A process ignoring SIGTERM must still die.
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
s.cfg.StopTimeout = 300 * time.Millisecond
|
||||||
|
|
||||||
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{},
|
||||||
|
shellCommand("trap '' TERM; sleep 30"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pid := sh.Status().PID
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
if err := s.Stop(t.Context(), "a"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if d := time.Since(start); d > 5*time.Second {
|
||||||
|
t.Errorf("stop took %v", d)
|
||||||
|
}
|
||||||
|
if processAlive(pid) {
|
||||||
|
t.Error("the process survived SIGKILL")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStartIsIdempotent(t *testing.T) {
|
||||||
|
// The control plane retries commands, so a repeated start must not produce
|
||||||
|
// a second process (plan §5.6).
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
|
||||||
|
first, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
second, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
|
||||||
|
if !errors.Is(err, ErrAlreadyRunning) {
|
||||||
|
t.Fatalf("expected ErrAlreadyRunning, got %v", err)
|
||||||
|
}
|
||||||
|
if first.Status().PID != second.Status().PID {
|
||||||
|
t.Error("a second process was started for the same shard")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRestartAfterTerminalState(t *testing.T) {
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
|
||||||
|
sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 1"))
|
||||||
|
waitState(t, sh, Failed, 3*time.Second)
|
||||||
|
|
||||||
|
// Once it has failed, the same id may be started again.
|
||||||
|
again, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("restarting a failed shard: %v", err)
|
||||||
|
}
|
||||||
|
if again.Status().State != Starting {
|
||||||
|
t.Errorf("state=%s", again.Status().State)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWaitReady(t *testing.T) {
|
||||||
|
s, h := newSupervisor(t)
|
||||||
|
|
||||||
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
h.ready.Store(true)
|
||||||
|
}()
|
||||||
|
|
||||||
|
if err := s.WaitReady(t.Context(), sh, "http://x", "m", 3*time.Second); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
st := sh.Status()
|
||||||
|
if st.State != Ready {
|
||||||
|
t.Errorf("state=%s", st.State)
|
||||||
|
}
|
||||||
|
if st.ReadyAt.IsZero() {
|
||||||
|
t.Error("ReadyAt was not recorded")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWaitReadyTimeoutIncludesLogs(t *testing.T) {
|
||||||
|
// When a model fails to load, the engine output is the only diagnosis
|
||||||
|
// available, so the error has to carry it.
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
|
||||||
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{},
|
||||||
|
shellCommand("echo 'CUDA out of memory'; sleep 30"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(100 * time.Millisecond) // let the line be captured
|
||||||
|
|
||||||
|
err = s.WaitReady(t.Context(), sh, "http://x", "m", 200*time.Millisecond)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected a timeout")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "CUDA out of memory") {
|
||||||
|
t.Errorf("the error does not carry engine output: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWaitReadyDetectsEarlyExit(t *testing.T) {
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
|
||||||
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 1"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Should return as soon as the process dies, not after the full timeout.
|
||||||
|
start := time.Now()
|
||||||
|
if err := s.WaitReady(t.Context(), sh, "http://x", "m", 10*time.Second); err == nil {
|
||||||
|
t.Fatal("expected an error")
|
||||||
|
}
|
||||||
|
if d := time.Since(start); d > 3*time.Second {
|
||||||
|
t.Errorf("waited %v for a process that had already exited", d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDrain(t *testing.T) {
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
|
||||||
|
sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("sleep 30"))
|
||||||
|
if err := s.Drain("a"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := sh.Status().State; got != Draining {
|
||||||
|
t.Errorf("state=%s", got)
|
||||||
|
}
|
||||||
|
// A drained shard's process is still alive; only routing stops.
|
||||||
|
if !processAlive(sh.Status().PID) {
|
||||||
|
t.Error("draining killed the process; it should keep serving in-flight work")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogsAreCaptured(t *testing.T) {
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
|
||||||
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{},
|
||||||
|
shellCommand("echo line1; echo line2 >&2; echo line3; sleep 0.2"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(400 * time.Millisecond)
|
||||||
|
|
||||||
|
logs := strings.Join(sh.Logs(10), "\n")
|
||||||
|
for _, want := range []string{"line1", "line2", "line3"} {
|
||||||
|
if !strings.Contains(logs, want) {
|
||||||
|
t.Errorf("%q missing from logs:\n%s", want, logs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- orphan adoption ---
|
||||||
|
|
||||||
|
func TestFindOrphans(t *testing.T) {
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
|
||||||
|
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20100}
|
||||||
|
sh, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pid := sh.Status().PID
|
||||||
|
|
||||||
|
// Simulate an agent restart: a new supervisor over the same work dir.
|
||||||
|
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
|
||||||
|
&scriptedHealth{}, slog.New(slog.DiscardHandler))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
orphans, err := s2.FindOrphans()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(orphans) != 1 {
|
||||||
|
t.Fatalf("expected 1 orphan, got %d", len(orphans))
|
||||||
|
}
|
||||||
|
o := orphans[0]
|
||||||
|
if o.ShardID != "a" || o.PID != pid || !o.Alive {
|
||||||
|
t.Errorf("orphan=%+v (want shard a, pid %d, alive)", o, pid)
|
||||||
|
}
|
||||||
|
// The command line is stored as a fingerprint, because pids get reused and
|
||||||
|
// adopting an unrelated process is worse than leaking the original.
|
||||||
|
if len(o.Cmdline) == 0 || !strings.Contains(strings.Join(o.Cmdline, " "), "sleep 30") {
|
||||||
|
t.Errorf("cmdline fingerprint missing: %v", o.Cmdline)
|
||||||
|
}
|
||||||
|
// And the spec, without which the control plane could not tell which
|
||||||
|
// replica the process belongs to and would place a duplicate.
|
||||||
|
if o.ReplicaID != "rep-1" {
|
||||||
|
t.Errorf("replica id not recovered: %q", o.ReplicaID)
|
||||||
|
}
|
||||||
|
if o.Spec.HTTPPort != 20100 {
|
||||||
|
t.Errorf("spec not recovered: %+v", o.Spec)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s2.KillOrphan(o); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(3 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if !processAlive(pid) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if processAlive(pid) {
|
||||||
|
t.Error("the orphan survived KillOrphan")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(s2.statePath("a")); !os.IsNotExist(err) {
|
||||||
|
t.Error("the state file was left behind")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStateFileRemovedOnExit(t *testing.T) {
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
|
||||||
|
sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 0"))
|
||||||
|
waitState(t, sh, Failed, 3*time.Second)
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
|
||||||
|
if _, err := os.Stat(s.statePath("a")); !os.IsNotExist(err) {
|
||||||
|
t.Error("the state file outlived the process; a later scan would find a dead orphan")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- state machine and ring buffer ---
|
||||||
|
|
||||||
|
func TestStateStrings(t *testing.T) {
|
||||||
|
for st, want := range map[State]string{
|
||||||
|
Pending: "pending", Starting: "starting", Loading: "loading",
|
||||||
|
Ready: "ready", Draining: "draining", Stopped: "stopped", Failed: "failed",
|
||||||
|
} {
|
||||||
|
if got := st.String(); got != want {
|
||||||
|
t.Errorf("%d -> %q, want %q", int(st), got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !Stopped.Terminal() || !Failed.Terminal() || Ready.Terminal() {
|
||||||
|
t.Error("Terminal is wrong")
|
||||||
|
}
|
||||||
|
if !Ready.Live() || !Draining.Live() || Stopped.Live() {
|
||||||
|
t.Error("Live is wrong")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRingBuffer(t *testing.T) {
|
||||||
|
r := newRingBuffer(3)
|
||||||
|
if got := r.tail(10); len(got) != 0 {
|
||||||
|
t.Errorf("empty buffer returned %v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
r.add("a")
|
||||||
|
r.add("b")
|
||||||
|
if got := strings.Join(r.tail(10), ","); got != "a,b" {
|
||||||
|
t.Errorf("tail=%q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
r.add("c")
|
||||||
|
r.add("d") // evicts "a"
|
||||||
|
if got := strings.Join(r.tail(10), ","); got != "b,c,d" {
|
||||||
|
t.Errorf("after wrap tail=%q, want b,c,d", got)
|
||||||
|
}
|
||||||
|
if got := strings.Join(r.tail(2), ","); got != "c,d" {
|
||||||
|
t.Errorf("tail(2)=%q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAdoptTakesOverRunningShard(t *testing.T) {
|
||||||
|
// The control plane restart fix depends on this: a shard that survived the
|
||||||
|
// agent must come back under supervision, not be restarted or leaked.
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
|
||||||
|
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20200}
|
||||||
|
sh, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pid := sh.Status().PID
|
||||||
|
|
||||||
|
// A new supervisor over the same work dir, as after an agent restart.
|
||||||
|
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 3 * time.Second},
|
||||||
|
&scriptedHealth{}, slog.New(slog.DiscardHandler))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
orphans, err := s2.FindOrphans()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(orphans) != 1 || !orphans[0].Alive {
|
||||||
|
t.Fatalf("orphan not found: %+v", orphans)
|
||||||
|
}
|
||||||
|
|
||||||
|
adopted, err := s2.Adopt(orphans[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := adopted.Status().PID; got != pid {
|
||||||
|
t.Errorf("adopted pid %d, expected %d", got, pid)
|
||||||
|
}
|
||||||
|
if adopted.Spec.ReplicaID != "rep-1" {
|
||||||
|
t.Errorf("spec lost in adoption: %+v", adopted.Spec)
|
||||||
|
}
|
||||||
|
// The earlier output is gone; saying so beats presenting an empty log as if
|
||||||
|
// the engine had been silent.
|
||||||
|
if logs := strings.Join(adopted.Logs(5), " "); !strings.Contains(logs, "adopted") {
|
||||||
|
t.Errorf("adoption was not noted in the log: %q", logs)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stopping an adopted shard must actually kill it: it has no exec.Cmd, so
|
||||||
|
// the signal path is the only one available.
|
||||||
|
if err := s2.Stop(t.Context(), "a"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if !processAlive(pid) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Error("the adopted process survived Stop")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAdoptRefusesDeadProcess(t *testing.T) {
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
if _, err := s.Adopt(Orphan{ShardID: "gone", PID: 999999, Alive: false}); err == nil {
|
||||||
|
t.Error("a dead process was adopted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAdoptedExitIsReported(t *testing.T) {
|
||||||
|
// An adopted engine dying has to be noticed; nothing waits on it.
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
|
||||||
|
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a"}
|
||||||
|
sh, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pid := sh.Status().PID
|
||||||
|
|
||||||
|
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
|
||||||
|
&scriptedHealth{}, slog.New(slog.DiscardHandler))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
orphans, _ := s2.FindOrphans()
|
||||||
|
adopted, err := s2.Adopt(orphans[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kill it behind the supervisor's back.
|
||||||
|
_ = killGroup(pid, syscall.SIGKILL)
|
||||||
|
waitState(t, adopted, Failed, 5*time.Second)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCorruptStateFileIsCleanedUp(t *testing.T) {
|
||||||
|
// A half-written state file describes a process that cannot be identified,
|
||||||
|
// so it cannot be adopted safely.
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
if err := os.WriteFile(s.statePath("broken"), []byte("{not json"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
orphans, err := s.FindOrphans()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(orphans) != 1 || orphans[0].ShardID != "broken" || orphans[0].Alive {
|
||||||
|
t.Fatalf("corrupt state file not reported for cleanup: %+v", orphans)
|
||||||
|
}
|
||||||
|
if err := s.KillOrphan(orphans[0]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(s.statePath("broken")); !os.IsNotExist(err) {
|
||||||
|
t.Error("the corrupt state file was left behind")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- multi-process shards (plan §11, phase 4) ---
|
||||||
|
|
||||||
|
func TestSidecarRunsBeforeEngine(t *testing.T) {
|
||||||
|
// The Ray shape: a supporting process starts first and keeps running, then
|
||||||
|
// the engine starts against it.
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
order := filepath.Join(dir, "order")
|
||||||
|
|
||||||
|
// Each writes to the order file and to stdout: the file records the
|
||||||
|
// sequence, stdout exercises per-process log attribution.
|
||||||
|
cmds := []engines.Command{
|
||||||
|
{Name: "sidecar", Path: "/bin/sh", Args: []string{"-c",
|
||||||
|
fmt.Sprintf("echo sidecar >> %s; echo hello-from-sidecar; sleep 30", order)}, Env: os.Environ()},
|
||||||
|
{Name: "engine", Path: "/bin/sh", Args: []string{"-c",
|
||||||
|
fmt.Sprintf("echo engine >> %s; echo hello-from-engine; sleep 30", order)}, Env: os.Environ()},
|
||||||
|
}
|
||||||
|
|
||||||
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, cmds)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generous, because this waits on two processes starting under the race
|
||||||
|
// detector while every other package is competing for the same cores. Three
|
||||||
|
// seconds was enough alone and not in a full run.
|
||||||
|
var started []string
|
||||||
|
deadline := time.Now().Add(20 * time.Second)
|
||||||
|
for time.Now().Before(deadline) && len(started) < 2 {
|
||||||
|
if b, err := os.ReadFile(order); err == nil {
|
||||||
|
started = strings.Fields(string(b))
|
||||||
|
}
|
||||||
|
if len(started) < 2 {
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Asserted rather than broken out of: the original fell through on timeout,
|
||||||
|
// so a slow machine turned the ordering check into a no-op and the failure
|
||||||
|
// surfaced later as something else.
|
||||||
|
if len(started) != 2 {
|
||||||
|
t.Fatalf("only %d of 2 processes started: %v", len(started), started)
|
||||||
|
}
|
||||||
|
if started[0] != "sidecar" || started[1] != "engine" {
|
||||||
|
t.Fatalf("start order was %v", started)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The shard's pid is the engine's, not the sidecar's: the engine's exit is
|
||||||
|
// what ends the shard.
|
||||||
|
if sh.Status().PID == 0 {
|
||||||
|
t.Fatal("no engine pid recorded")
|
||||||
|
}
|
||||||
|
// Both processes appear in one log tail, distinguishable by prefix.
|
||||||
|
deadline = time.Now().Add(20 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
logs := strings.Join(sh.Logs(20), "\n")
|
||||||
|
if strings.Contains(logs, "[sidecar] hello-from-sidecar") &&
|
||||||
|
strings.Contains(logs, "[engine] hello-from-engine") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Errorf("output is not attributed per process:\n%s", strings.Join(sh.Logs(20), "\n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStopKillsSidecar(t *testing.T) {
|
||||||
|
// A Ray head left behind holds the port the next replica needs, and would
|
||||||
|
// let a later shard join a cluster that serves nothing.
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
marker := filepath.Join(dir, "sidecar.pid")
|
||||||
|
|
||||||
|
cmds := []engines.Command{
|
||||||
|
{Name: "sidecar", Path: "/bin/sh", Args: []string{"-c",
|
||||||
|
fmt.Sprintf("echo $$ > %s; sleep 30", marker)}, Env: os.Environ()},
|
||||||
|
{Name: "engine", Path: "/bin/sh", Args: []string{"-c", "sleep 30"}, Env: os.Environ()},
|
||||||
|
}
|
||||||
|
if _, err := s.Start(t.Context(), "a", engines.ShardSpec{}, cmds); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var sidePID int
|
||||||
|
deadline := time.Now().Add(3 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if b, err := os.ReadFile(marker); err == nil {
|
||||||
|
if _, err := fmt.Sscanf(strings.TrimSpace(string(b)), "%d", &sidePID); err == nil && sidePID > 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if sidePID == 0 {
|
||||||
|
t.Fatal("the sidecar never started")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.Stop(t.Context(), "a"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
deadline = time.Now().Add(3 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if !processAlive(sidePID) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Errorf("the sidecar (%d) survived the stop and still holds its port", sidePID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSidecarFailureFailsTheShard(t *testing.T) {
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
cmds := []engines.Command{
|
||||||
|
{Name: "sidecar", Path: "/nonexistent/binary", Env: os.Environ()},
|
||||||
|
{Name: "engine", Path: "/bin/sh", Args: []string{"-c", "sleep 30"}, Env: os.Environ()},
|
||||||
|
}
|
||||||
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{}, cmds)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a shard whose sidecar could not start was reported healthy")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "sidecar") {
|
||||||
|
t.Errorf("the error does not identify the sidecar: %v", err)
|
||||||
|
}
|
||||||
|
if sh.Status().State != Failed {
|
||||||
|
t.Errorf("state=%s", sh.Status().State)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWaitForLeaderPortBlocksUntilListening(t *testing.T) {
|
||||||
|
// A Ray worker cannot join a head that is not up, and the leader's server
|
||||||
|
// cannot claim GPUs from a cluster that does not exist yet.
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
s.cfg.RendezvousTimeout = 5 * time.Second
|
||||||
|
|
||||||
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
port := ln.Addr().(*net.TCPAddr).Port
|
||||||
|
_ = ln.Close() // nothing listening yet
|
||||||
|
|
||||||
|
spec := engines.ShardSpec{LeaderAddr: "127.0.0.1", LeaderPort: port}
|
||||||
|
cmds := []engines.Command{
|
||||||
|
{Name: "sidecar", Path: "/bin/sh", Args: []string{"-c",
|
||||||
|
fmt.Sprintf("sleep 0.4; nc -l %d", port)}, Env: os.Environ(),
|
||||||
|
WaitForLeaderPort: true},
|
||||||
|
{Name: "engine", Path: "/bin/sh", Args: []string{"-c", "sleep 30"}, Env: os.Environ()},
|
||||||
|
}
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
if _, err := s.Start(t.Context(), "a", spec, cmds); err != nil {
|
||||||
|
t.Skipf("nc is unavailable for the rendezvous stub: %v", err)
|
||||||
|
}
|
||||||
|
if d := time.Since(start); d < 300*time.Millisecond {
|
||||||
|
t.Errorf("the engine started after %v, before the rendezvous point was listening", d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRendezvousTimeout(t *testing.T) {
|
||||||
|
// Nothing ever listens, so the shard must fail rather than hang forever.
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
s.cfg.RendezvousTimeout = 300 * time.Millisecond
|
||||||
|
|
||||||
|
spec := engines.ShardSpec{LeaderAddr: "127.0.0.1", LeaderPort: 1}
|
||||||
|
cmds := []engines.Command{
|
||||||
|
{Name: "sidecar", Path: "/bin/sh", Args: []string{"-c", "sleep 30"},
|
||||||
|
Env: os.Environ(), WaitForLeaderPort: true},
|
||||||
|
{Name: "engine", Path: "/bin/sh", Args: []string{"-c", "sleep 30"}, Env: os.Environ()},
|
||||||
|
}
|
||||||
|
if _, err := s.Start(t.Context(), "a", spec, cmds); err == nil {
|
||||||
|
t.Fatal("the shard did not fail when the rendezvous point never appeared")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWaitAliveForNonServingRank(t *testing.T) {
|
||||||
|
// A worker has no endpoint to poll, so staying up is the only evidence
|
||||||
|
// available. This is what stopped multi-node replicas coming up at all.
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
s.cfg.AliveGrace = 150 * time.Millisecond
|
||||||
|
|
||||||
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{Rank: 1}, shellCommand("sleep 30"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := s.WaitAlive(t.Context(), sh); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := sh.Status().State; got != Ready {
|
||||||
|
t.Errorf("state=%s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWaitAliveFailsOnEarlyExit(t *testing.T) {
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
s.cfg.AliveGrace = 2 * time.Second
|
||||||
|
|
||||||
|
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{Rank: 1}, shellCommand("exit 1"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
start := time.Now()
|
||||||
|
if err := s.WaitAlive(t.Context(), sh); err == nil {
|
||||||
|
t.Fatal("a worker that died immediately was reported ready")
|
||||||
|
}
|
||||||
|
if d := time.Since(start); d > 1500*time.Millisecond {
|
||||||
|
t.Errorf("waited %v for a process that had already exited", d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAdoptedShardCanBecomeReady(t *testing.T) {
|
||||||
|
// Adoption stopped at "the process is alive", leaving the shard in Starting
|
||||||
|
// with nothing that would ever move it. The control plane then waited out
|
||||||
|
// the startup timeout and killed a perfectly healthy engine to start an
|
||||||
|
// identical one — so an agent restart, which exists precisely so engines
|
||||||
|
// are not interrupted, cost every model on the node a full reload.
|
||||||
|
//
|
||||||
|
// Seen on node05: the engine answered /health throughout and the replica
|
||||||
|
// sat in Starting for three minutes until the timeout reaped it.
|
||||||
|
s, _ := newSupervisor(t)
|
||||||
|
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20201}
|
||||||
|
if _, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
health := &scriptedHealth{}
|
||||||
|
s2, err := New(Config{
|
||||||
|
WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second,
|
||||||
|
HealthInterval: 10 * time.Millisecond,
|
||||||
|
}, health, slog.New(slog.DiscardHandler))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
orphans, err := s2.FindOrphans()
|
||||||
|
if err != nil || len(orphans) != 1 {
|
||||||
|
t.Fatalf("orphan not found: %+v (%v)", orphans, err)
|
||||||
|
}
|
||||||
|
adopted, err := s2.Adopt(orphans[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := adopted.Status().State; got != Starting {
|
||||||
|
t.Fatalf("a freshly adopted shard is %s, expected Starting", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The engine answers, as node05's did. The shard must be able to reach
|
||||||
|
// Ready from here; before this it had no path to it at all.
|
||||||
|
health.ready.Store(true)
|
||||||
|
if err := s2.WaitReady(t.Context(), adopted, "http://127.0.0.1:20201", "m", 2*time.Second); err != nil {
|
||||||
|
t.Fatalf("an adopted, healthy engine never became ready: %v", err)
|
||||||
|
}
|
||||||
|
if got := adopted.Status().State; got != Ready {
|
||||||
|
t.Errorf("state %s after a passing health check", got)
|
||||||
|
}
|
||||||
|
_ = s2.Stop(t.Context(), "a")
|
||||||
|
}
|
||||||
@@ -0,0 +1,309 @@
|
|||||||
|
// Package weights makes a model's files present on a node.
|
||||||
|
//
|
||||||
|
// The simple arrangement, and the default: every node has its own copy. A
|
||||||
|
// source that is already a local path is used as it stands; one that names a
|
||||||
|
// remote file is fetched into a cache directory once and reused. Nothing here
|
||||||
|
// manages storage, decides a retention policy, or knows what a filesystem is —
|
||||||
|
// it answers one question, which is where the engine should be pointed.
|
||||||
|
//
|
||||||
|
// The exotic arrangements are all handled by not acting. Weights on a shared
|
||||||
|
// filer, an image exported over the fabric, or anything else an operator has
|
||||||
|
// mounted are simply present, and present means there is nothing to do. That is
|
||||||
|
// why the interesting case needs no code: the escape hatch from a distribution
|
||||||
|
// mechanism is a distribution mechanism that checks first.
|
||||||
|
package weights
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Progress reports how a fetch is going, for anything that wants to say so.
|
||||||
|
type Progress struct {
|
||||||
|
Source string
|
||||||
|
BytesDone int64
|
||||||
|
BytesTotal int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Percent is how far along, or -1 when the total is unknown.
|
||||||
|
func (p Progress) Percent() int {
|
||||||
|
if p.BytesTotal <= 0 {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
return int(p.BytesDone * 100 / p.BytesTotal)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fetcher makes sources local.
|
||||||
|
type Fetcher struct {
|
||||||
|
CacheDir string
|
||||||
|
Client *http.Client
|
||||||
|
Log *slog.Logger
|
||||||
|
// OnProgress is called occasionally during a fetch. A missing model is the
|
||||||
|
// difference between a replica that is slow to start and one that is stuck,
|
||||||
|
// and without this the two are indistinguishable from outside.
|
||||||
|
OnProgress func(Progress)
|
||||||
|
|
||||||
|
// inflight makes concurrent requests for one source share a single fetch.
|
||||||
|
//
|
||||||
|
// Eight shards of a tensor-parallel replica start at the same instant and
|
||||||
|
// want the same file. Without this they would each open a connection and
|
||||||
|
// each write the same bytes, turning one download into eight and, on a
|
||||||
|
// shared origin, into a self-inflicted denial of service.
|
||||||
|
mu sync.Mutex
|
||||||
|
inflight map[string]*fetchCall
|
||||||
|
}
|
||||||
|
|
||||||
|
type fetchCall struct {
|
||||||
|
done chan struct{}
|
||||||
|
path string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrNotLocal means a source names a file that is not here and cannot be
|
||||||
|
// fetched — a bare path with nothing to fetch it from.
|
||||||
|
var ErrNotLocal = errors.New("the weights are not on this node and the source says nothing about where to get them")
|
||||||
|
|
||||||
|
// Ensure returns a local path for a source, fetching it if it is not here.
|
||||||
|
//
|
||||||
|
// A plain path is never fetched, only checked. That is deliberate: a path is
|
||||||
|
// what an operator writes when the file is already there, whether it arrived on
|
||||||
|
// local disk, a Lustre mount or an image exported over the fabric, and a
|
||||||
|
// downloader that second-guessed that would copy a hundred gigabytes that were
|
||||||
|
// already present.
|
||||||
|
func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||||
|
if source == "" {
|
||||||
|
return "", fmt.Errorf("no source given")
|
||||||
|
}
|
||||||
|
if !isRemote(source) {
|
||||||
|
if !isPath(source) {
|
||||||
|
// A bare model id — "Qwen/Qwen2.5-7B-Instruct" — is not a file and
|
||||||
|
// was never meant to be one. vLLM and SGLang resolve those
|
||||||
|
// themselves, and doing it for them would be both a duplicate
|
||||||
|
// downloader and a change in documented behaviour.
|
||||||
|
return source, nil
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(source); err == nil {
|
||||||
|
return source, nil
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("%w: %s", ErrNotLocal, source)
|
||||||
|
}
|
||||||
|
|
||||||
|
target, err := f.cachePath(source)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(target); err == nil {
|
||||||
|
return target, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
f.mu.Lock()
|
||||||
|
if f.inflight == nil {
|
||||||
|
f.inflight = map[string]*fetchCall{}
|
||||||
|
}
|
||||||
|
if call, running := f.inflight[source]; running {
|
||||||
|
f.mu.Unlock()
|
||||||
|
select {
|
||||||
|
case <-call.done:
|
||||||
|
return call.path, call.err
|
||||||
|
case <-ctx.Done():
|
||||||
|
return "", ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
call := &fetchCall{done: make(chan struct{})}
|
||||||
|
f.inflight[source] = call
|
||||||
|
f.mu.Unlock()
|
||||||
|
|
||||||
|
call.path, call.err = f.fetch(ctx, source, target)
|
||||||
|
close(call.done)
|
||||||
|
|
||||||
|
f.mu.Lock()
|
||||||
|
delete(f.inflight, source)
|
||||||
|
f.mu.Unlock()
|
||||||
|
return call.path, call.err
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetch downloads a source to a target, resuming and verifying.
|
||||||
|
func (f *Fetcher) fetch(ctx context.Context, source, target string) (string, error) {
|
||||||
|
url, err := resolveURL(source)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Written beside the target and renamed at the end. A partial file under
|
||||||
|
// the real name would be found by the next scan, reported as present, and
|
||||||
|
// handed to an engine that fails on it minutes later — the exact failure
|
||||||
|
// this whole area exists to prevent.
|
||||||
|
part := target + ".part"
|
||||||
|
existing := int64(0)
|
||||||
|
if st, err := os.Stat(part); err == nil {
|
||||||
|
existing = st.Size()
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if existing > 0 {
|
||||||
|
// Resume. A forty gigabyte fetch that fails at ninety percent must not
|
||||||
|
// start again from nothing.
|
||||||
|
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", existing))
|
||||||
|
}
|
||||||
|
|
||||||
|
client := f.Client
|
||||||
|
if client == nil {
|
||||||
|
client = &http.Client{}
|
||||||
|
}
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("fetching %s: %w", source, err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
switch resp.StatusCode {
|
||||||
|
case http.StatusOK:
|
||||||
|
existing = 0 // the server ignored the range; start over
|
||||||
|
case http.StatusPartialContent:
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("fetching %s: %s returned %s", source, url, resp.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
flags := os.O_CREATE | os.O_WRONLY
|
||||||
|
if existing > 0 {
|
||||||
|
flags |= os.O_APPEND
|
||||||
|
} else {
|
||||||
|
flags |= os.O_TRUNC
|
||||||
|
}
|
||||||
|
out, err := os.OpenFile(part, flags, 0o644)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
total := existing + resp.ContentLength
|
||||||
|
f.report(Progress{Source: source, BytesDone: existing, BytesTotal: total})
|
||||||
|
written, err := f.copyWithProgress(ctx, out, resp.Body, source, existing, total)
|
||||||
|
closeErr := out.Close()
|
||||||
|
if err != nil {
|
||||||
|
// The part file is kept on purpose, so the next attempt resumes.
|
||||||
|
return "", fmt.Errorf("fetching %s after %d bytes: %w", source, written, err)
|
||||||
|
}
|
||||||
|
if closeErr != nil {
|
||||||
|
return "", closeErr
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.Rename(part, target); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if f.Log != nil {
|
||||||
|
f.Log.Info("weights fetched", "source", source, "path", target, "bytes", written)
|
||||||
|
}
|
||||||
|
return target, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *Fetcher) copyWithProgress(ctx context.Context, dst io.Writer, src io.Reader,
|
||||||
|
source string, done, total int64,
|
||||||
|
) (int64, error) {
|
||||||
|
buf := make([]byte, 4<<20)
|
||||||
|
last := time.Now()
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return done, err
|
||||||
|
}
|
||||||
|
n, err := src.Read(buf)
|
||||||
|
if n > 0 {
|
||||||
|
if _, werr := dst.Write(buf[:n]); werr != nil {
|
||||||
|
return done, werr
|
||||||
|
}
|
||||||
|
done += int64(n)
|
||||||
|
// Occasionally, not per read: a four-megabyte buffer over a
|
||||||
|
// hundred gigabytes is twenty-five thousand callbacks, and nobody
|
||||||
|
// needs to be told that often.
|
||||||
|
if time.Since(last) > 2*time.Second {
|
||||||
|
f.report(Progress{Source: source, BytesDone: done, BytesTotal: total})
|
||||||
|
last = time.Now()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
f.report(Progress{Source: source, BytesDone: done, BytesTotal: total})
|
||||||
|
return done, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return done, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *Fetcher) report(p Progress) {
|
||||||
|
if f.OnProgress != nil {
|
||||||
|
f.OnProgress(p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// cachePath is where a remote source lands.
|
||||||
|
//
|
||||||
|
// Named by a hash of the source with the original filename kept on the end, so
|
||||||
|
// two models with the same filename from different places cannot collide while
|
||||||
|
// a human looking at the directory can still tell what is in it.
|
||||||
|
func (f *Fetcher) cachePath(source string) (string, error) {
|
||||||
|
if f.CacheDir == "" {
|
||||||
|
return "", fmt.Errorf("no cache directory configured, so %s cannot be fetched", source)
|
||||||
|
}
|
||||||
|
return CachePathFor(f.CacheDir, source), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// isPath reports whether a source names a file on this machine.
|
||||||
|
//
|
||||||
|
// By its leading character, because that is the only honest signal available:
|
||||||
|
// "Qwen/Qwen2.5-7B-Instruct" and "models/qwen.gguf" are indistinguishable as
|
||||||
|
// strings, and an operator writing a local file writes an absolute path. The
|
||||||
|
// cost of being wrong in this direction is that a relative path is passed to
|
||||||
|
// the engine untouched, which is what happened before this code existed.
|
||||||
|
func isPath(source string) bool {
|
||||||
|
return strings.HasPrefix(source, "/") ||
|
||||||
|
strings.HasPrefix(source, "./") ||
|
||||||
|
strings.HasPrefix(source, "../")
|
||||||
|
}
|
||||||
|
|
||||||
|
// isRemote reports whether a source has to be fetched at all.
|
||||||
|
func isRemote(source string) bool {
|
||||||
|
return strings.HasPrefix(source, "hf:") ||
|
||||||
|
strings.HasPrefix(source, "http://") ||
|
||||||
|
strings.HasPrefix(source, "https://")
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveURL turns a source into something to GET.
|
||||||
|
//
|
||||||
|
// hf: names one file in a repository, which is what a GGUF is. A whole
|
||||||
|
// repository of shards is deliberately not handled here: vLLM and SGLang
|
||||||
|
// download those themselves and do it well, and reimplementing a snapshot
|
||||||
|
// downloader to compete with them would be work spent to arrive back where we
|
||||||
|
// started.
|
||||||
|
func resolveURL(source string) (string, error) {
|
||||||
|
if rest, ok := strings.CutPrefix(source, "hf:"); ok {
|
||||||
|
parts := strings.Split(rest, "/")
|
||||||
|
if len(parts) < 3 {
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"%s names a repository, not a file; give the file as hf:org/repo/model.gguf, "+
|
||||||
|
"or use the repository id directly and let the engine download it", source)
|
||||||
|
}
|
||||||
|
repo := strings.Join(parts[:2], "/")
|
||||||
|
file := strings.Join(parts[2:], "/")
|
||||||
|
return "https://huggingface.co/" + repo + "/resolve/main/" + file, nil
|
||||||
|
}
|
||||||
|
if _, err := url.Parse(source); err != nil {
|
||||||
|
return "", fmt.Errorf("%s is not a usable source: %w", source, err)
|
||||||
|
}
|
||||||
|
return source, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,289 @@
|
|||||||
|
package weights
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func serveBytes(t *testing.T, body []byte, hits *atomic.Int64) *httptest.Server {
|
||||||
|
t.Helper()
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if hits != nil {
|
||||||
|
hits.Add(1)
|
||||||
|
}
|
||||||
|
http.ServeContent(w, r, "model.gguf", time.Time{}, strings.NewReader(string(body)))
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestALocalPathIsUsedNeverCopied(t *testing.T) {
|
||||||
|
// A path is what an operator writes when the file is already there —
|
||||||
|
// local disk, a Lustre mount, an image exported over the fabric. A
|
||||||
|
// downloader that second-guessed that would copy a hundred gigabytes that
|
||||||
|
// were already present.
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "model.gguf")
|
||||||
|
if err := os.WriteFile(path, []byte("weights"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f := &Fetcher{CacheDir: t.TempDir()}
|
||||||
|
|
||||||
|
got, err := f.Ensure(t.Context(), path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got != path {
|
||||||
|
t.Errorf("returned %q, want the original path", got)
|
||||||
|
}
|
||||||
|
if entries, _ := os.ReadDir(f.CacheDir); len(entries) != 0 {
|
||||||
|
t.Errorf("the cache was written to for a local file: %v", entries)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMissingLocalPathSaysWhatIsWrong(t *testing.T) {
|
||||||
|
// The commonest setup mistake, and "no such file" from an engine twenty
|
||||||
|
// minutes into a load is a poor way to learn it.
|
||||||
|
f := &Fetcher{CacheDir: t.TempDir()}
|
||||||
|
_, err := f.Ensure(t.Context(), filepath.Join(t.TempDir(), "absent.gguf"))
|
||||||
|
if !errors.Is(err, ErrNotLocal) {
|
||||||
|
t.Fatalf("err = %v, want ErrNotLocal", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "absent.gguf") {
|
||||||
|
t.Errorf("the error does not name the file: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARemoteFileIsFetchedOnceAndReused(t *testing.T) {
|
||||||
|
var hits atomic.Int64
|
||||||
|
srv := serveBytes(t, []byte("the weights"), &hits)
|
||||||
|
f := &Fetcher{CacheDir: t.TempDir()}
|
||||||
|
|
||||||
|
first, err := f.Ensure(t.Context(), srv.URL+"/model.gguf")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, err := os.ReadFile(first)
|
||||||
|
if err != nil || string(body) != "the weights" {
|
||||||
|
t.Fatalf("fetched content = %q, err %v", body, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
second, err := f.Ensure(t.Context(), srv.URL+"/model.gguf")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if second != first {
|
||||||
|
t.Errorf("the second call returned a different path: %q vs %q", second, first)
|
||||||
|
}
|
||||||
|
if hits.Load() != 1 {
|
||||||
|
t.Errorf("the file was fetched %d times; the cache did not hold", hits.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConcurrentRequestsProduceOneDownload(t *testing.T) {
|
||||||
|
// Eight shards of a tensor-parallel replica start at the same instant and
|
||||||
|
// want the same file. Eight downloads is a self-inflicted denial of service
|
||||||
|
// on whatever is serving them.
|
||||||
|
var hits atomic.Int64
|
||||||
|
srv := serveBytes(t, []byte("shared weights"), &hits)
|
||||||
|
f := &Fetcher{CacheDir: t.TempDir()}
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
paths := make([]string, 8)
|
||||||
|
errs := make([]error, 8)
|
||||||
|
for i := range 8 {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
paths[i], errs[i] = f.Ensure(t.Context(), srv.URL+"/model.gguf")
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
for i, err := range errs {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("caller %d failed: %v", i, err)
|
||||||
|
}
|
||||||
|
if paths[i] != paths[0] {
|
||||||
|
t.Errorf("caller %d got %q, caller 0 got %q", i, paths[i], paths[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n := hits.Load(); n != 1 {
|
||||||
|
t.Errorf("%d downloads for eight concurrent callers", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPartialDownloadIsNeverVisibleAsTheModel(t *testing.T) {
|
||||||
|
// A partial file under the real name would be found by the next scan,
|
||||||
|
// reported as present, and handed to an engine that fails on it minutes
|
||||||
|
// later — the exact failure all of this exists to prevent.
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Length", "1000")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
_, _ = w.Write([]byte("only the first bytes"))
|
||||||
|
// and then the connection dies without the rest
|
||||||
|
if fl, ok := w.(http.Flusher); ok {
|
||||||
|
fl.Flush()
|
||||||
|
}
|
||||||
|
panic(http.ErrAbortHandler)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
cache := t.TempDir()
|
||||||
|
f := &Fetcher{CacheDir: cache}
|
||||||
|
if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf"); err == nil {
|
||||||
|
t.Fatal("a truncated download was reported as success")
|
||||||
|
}
|
||||||
|
entries, _ := os.ReadDir(cache)
|
||||||
|
for _, e := range entries {
|
||||||
|
if !strings.HasSuffix(e.Name(), ".part") {
|
||||||
|
t.Errorf("a non-part file was left behind: %q", e.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnInterruptedDownloadResumes(t *testing.T) {
|
||||||
|
// A forty gigabyte fetch that fails at ninety percent must not start again
|
||||||
|
// from nothing.
|
||||||
|
full := []byte(strings.Repeat("x", 4096))
|
||||||
|
var ranges []string
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ranges = append(ranges, r.Header.Get("Range"))
|
||||||
|
http.ServeContent(w, r, "model.gguf", time.Time{}, strings.NewReader(string(full)))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
cache := t.TempDir()
|
||||||
|
f := &Fetcher{CacheDir: cache}
|
||||||
|
target, _ := f.cachePath(srv.URL + "/model.gguf")
|
||||||
|
// Half a file already on disk, as an interrupted attempt would leave.
|
||||||
|
if err := os.WriteFile(target+".part", full[:2048], 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := f.Ensure(t.Context(), srv.URL+"/model.gguf")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, _ := os.ReadFile(got)
|
||||||
|
if len(body) != len(full) {
|
||||||
|
t.Errorf("resumed file is %d bytes, want %d", len(body), len(full))
|
||||||
|
}
|
||||||
|
if len(ranges) == 0 || !strings.Contains(ranges[0], "2048") {
|
||||||
|
t.Errorf("no range request was made: %v", ranges)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnHFSourceNamingARepositoryExplainsItself(t *testing.T) {
|
||||||
|
// A whole repository of shards is deliberately not handled here: vLLM and
|
||||||
|
// SGLang download those themselves and do it well. Saying so beats a
|
||||||
|
// confusing 404.
|
||||||
|
_, err := resolveURL("hf:Qwen/Qwen2.5-Coder-32B-Instruct")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a bare repository was accepted as a file")
|
||||||
|
}
|
||||||
|
for _, want := range []string{"model.gguf", "let the engine"} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("the error does not suggest what to do (%q): %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnHFFileResolvesToItsDownloadURL(t *testing.T) {
|
||||||
|
got, err := resolveURL("hf:Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/qwen2.5-coder-3b-instruct-q4_k_m.gguf")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := "https://huggingface.co/Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/resolve/main/qwen2.5-coder-3b-instruct-q4_k_m.gguf"
|
||||||
|
if got != want {
|
||||||
|
t.Errorf("resolved to\n %s\nwant\n %s", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTwoSourcesWithTheSameFilenameDoNotCollide(t *testing.T) {
|
||||||
|
f := &Fetcher{CacheDir: t.TempDir()}
|
||||||
|
a, _ := f.cachePath("https://one.example/model.gguf")
|
||||||
|
b, _ := f.cachePath("https://two.example/model.gguf")
|
||||||
|
if a == b {
|
||||||
|
t.Fatalf("both sources cache to %q", a)
|
||||||
|
}
|
||||||
|
// And a person looking at the directory can still tell what is in it.
|
||||||
|
for _, p := range []string{a, b} {
|
||||||
|
if !strings.HasSuffix(p, "model.gguf") {
|
||||||
|
t.Errorf("cache path %q does not keep the filename", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProgressIsReportedWithAPercentage(t *testing.T) {
|
||||||
|
// A missing model is the difference between a replica that is slow to start
|
||||||
|
// and one that is stuck, and without this they look identical.
|
||||||
|
srv := serveBytes(t, []byte(strings.Repeat("y", 8192)), nil)
|
||||||
|
var seen []Progress
|
||||||
|
f := &Fetcher{
|
||||||
|
CacheDir: t.TempDir(),
|
||||||
|
OnProgress: func(p Progress) { seen = append(seen, p) },
|
||||||
|
}
|
||||||
|
if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(seen) == 0 {
|
||||||
|
t.Fatal("no progress was reported")
|
||||||
|
}
|
||||||
|
last := seen[len(seen)-1]
|
||||||
|
if last.Percent() != 100 {
|
||||||
|
t.Errorf("final progress is %d%%, want 100 (%+v)", last.Percent(), last)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPercentIsUnknownRatherThanZeroWithoutATotal(t *testing.T) {
|
||||||
|
// A server that sends no Content-Length is common, and reporting 0% for
|
||||||
|
// a download that is nearly finished is worse than admitting ignorance.
|
||||||
|
if got := (Progress{BytesDone: 500}).Percent(); got != -1 {
|
||||||
|
t.Errorf("Percent with no total = %d, want -1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoCacheDirectoryIsAClearRefusal(t *testing.T) {
|
||||||
|
f := &Fetcher{}
|
||||||
|
_, err := f.Ensure(t.Context(), "https://example.invalid/model.gguf")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "cache directory") {
|
||||||
|
t.Errorf("err = %v, want a complaint about the cache directory", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestABareModelIdIsLeftToTheEngine(t *testing.T) {
|
||||||
|
// "Qwen/Qwen2.5-7B-Instruct" is a documented source that vLLM and SGLang
|
||||||
|
// resolve themselves. Treating it as a missing file broke every existing
|
||||||
|
// vLLM profile, which is how this was found.
|
||||||
|
f := &Fetcher{CacheDir: t.TempDir()}
|
||||||
|
for _, id := range []string{
|
||||||
|
"Qwen/Qwen2.5-7B-Instruct",
|
||||||
|
"meta-llama/Llama-3.3-70B-Instruct",
|
||||||
|
} {
|
||||||
|
got, err := f.Ensure(t.Context(), id)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("%s was refused: %v", id, err)
|
||||||
|
}
|
||||||
|
if got != id {
|
||||||
|
t.Errorf("%s was rewritten to %q", id, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAbsolutePathThatIsMissingIsStillAnError(t *testing.T) {
|
||||||
|
// The other half: an operator writing a path means a file, and a missing
|
||||||
|
// one should be said plainly rather than handed to an engine.
|
||||||
|
f := &Fetcher{CacheDir: t.TempDir()}
|
||||||
|
if _, err := f.Ensure(t.Context(), "/models/definitely-not-here.gguf"); !errors.Is(err, ErrNotLocal) {
|
||||||
|
t.Errorf("err = %v, want ErrNotLocal", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
package weights
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// cacheNamePattern matches a filename this package created.
|
||||||
|
//
|
||||||
|
// The eight-byte hash of the source, hex encoded, then a dash, then the
|
||||||
|
// original filename — exactly what cachePath writes. It is the only mark of
|
||||||
|
// ownership there is, and it has to be exact: everything else in a model
|
||||||
|
// directory was put there by the operator, and deleting one of those files
|
||||||
|
// could cost a week of someone's bandwidth.
|
||||||
|
var cacheNamePattern = regexp.MustCompile(`^[0-9a-f]{16}-`)
|
||||||
|
|
||||||
|
// CachePathFor is where a source would be cached in the given directory.
|
||||||
|
//
|
||||||
|
// Exported so the control plane can work out which files in a node's reported
|
||||||
|
// list are ones llmgw fetched for a model it still wants, without asking the
|
||||||
|
// node anything. The agent and the control plane must agree on this name
|
||||||
|
// exactly, which is why there is one function and not two.
|
||||||
|
func CachePathFor(dir, source string) string {
|
||||||
|
sum := sha256.Sum256([]byte(source))
|
||||||
|
name := filepath.Base(source)
|
||||||
|
if name == "" || name == "." || name == "/" {
|
||||||
|
name = "weights"
|
||||||
|
}
|
||||||
|
return filepath.Join(dir, hex.EncodeToString(sum[:8])+"-"+name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Owned reports whether a path is one this package created.
|
||||||
|
//
|
||||||
|
// The dividing line of the whole feature. A file without the cache name was
|
||||||
|
// put there by the operator — by scp, by a shared mount, by hand — and llmgw
|
||||||
|
// has no business deleting it however unreferenced it looks. A file with the
|
||||||
|
// cache name is one llmgw downloaded, and llmgw can take it back.
|
||||||
|
func Owned(path string) bool {
|
||||||
|
return cacheNamePattern.MatchString(filepath.Base(path))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Orphans lists cached files that no wanted source accounts for.
|
||||||
|
//
|
||||||
|
// `have` is what a node reports on disk and `wanted` is every source in the
|
||||||
|
// catalog; both are passed in so this is a calculation rather than a scan, and
|
||||||
|
// so it can be run on the control plane against a node's reported inventory.
|
||||||
|
//
|
||||||
|
// Only files this package created are ever returned. A source that is not
|
||||||
|
// fetched — a plain path, or a model id an engine resolves itself — produces
|
||||||
|
// no cache name and so protects nothing and deletes nothing, which is correct:
|
||||||
|
// those files are not in the cache to begin with.
|
||||||
|
func Orphans(have []string, wanted []string) []string {
|
||||||
|
keep := map[string]struct{}{}
|
||||||
|
for _, src := range wanted {
|
||||||
|
if !isRemote(src) {
|
||||||
|
// Nothing was fetched for it, so nothing in the cache belongs to
|
||||||
|
// it. Computing a name here would reserve a file that does not
|
||||||
|
// exist and could never be an orphan anyway.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// By base name rather than full path: the control plane does not know
|
||||||
|
// each node's cache directory, and the name is what identifies the
|
||||||
|
// file wherever the node keeps it.
|
||||||
|
keep[filepath.Base(CachePathFor("", src))] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
var out []string
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
for _, path := range have {
|
||||||
|
base := filepath.Base(path)
|
||||||
|
if !Owned(base) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, wanted := keep[base]; wanted {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, dup := seen[path]; dup {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[path] = struct{}{}
|
||||||
|
out = append(out, path)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// PartialName reports whether a path is the temporary file a fetch writes to.
|
||||||
|
//
|
||||||
|
// Those are not orphans in the same sense — a fetch may be running right now —
|
||||||
|
// so they are named separately rather than swept up with the rest. A `.part`
|
||||||
|
// left behind by a killed agent is real garbage, but only its own node can
|
||||||
|
// tell whether it is stale, and nothing here can.
|
||||||
|
func PartialName(path string) bool {
|
||||||
|
return strings.HasSuffix(path, ".part")
|
||||||
|
}
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
package weights
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestOnlyFilesLLMGWFetchedAreEverOrphans(t *testing.T) {
|
||||||
|
// The invariant the whole feature rests on. Everything without the cache
|
||||||
|
// name was put there by the operator — scp, a shared mount, a copy by hand
|
||||||
|
// — and deleting one could cost a week of someone's bandwidth. It does not
|
||||||
|
// matter how unreferenced it looks.
|
||||||
|
const dir = "/var/lib/llmgw-models"
|
||||||
|
have := []string{
|
||||||
|
filepath.Join(dir, "qwen2.5-coder-14b-instruct-q8_0-00001-of-00002.gguf"),
|
||||||
|
filepath.Join(dir, "some-model.safetensors"),
|
||||||
|
filepath.Join(dir, "notes.txt"),
|
||||||
|
CachePathFor(dir, "https://models.internal/gone.gguf"),
|
||||||
|
}
|
||||||
|
|
||||||
|
got := Orphans(have, nil)
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("expected only the fetched file, got %v", got)
|
||||||
|
}
|
||||||
|
if !Owned(got[0]) {
|
||||||
|
t.Errorf("returned a file llmgw did not fetch: %s", got[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFetchedFileStillInTheCatalogIsKept(t *testing.T) {
|
||||||
|
const dir = "/var/lib/llmgw-models"
|
||||||
|
const wanted = "hf:Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/q4.gguf"
|
||||||
|
const gone = "https://models.internal/old.gguf"
|
||||||
|
|
||||||
|
have := []string{CachePathFor(dir, wanted), CachePathFor(dir, gone)}
|
||||||
|
got := Orphans(have, []string{wanted})
|
||||||
|
|
||||||
|
if slices.Contains(got, CachePathFor(dir, wanted)) {
|
||||||
|
t.Error("a file the catalog still points at was called an orphan")
|
||||||
|
}
|
||||||
|
if !slices.Contains(got, CachePathFor(dir, gone)) {
|
||||||
|
t.Errorf("a file nothing points at was not found: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheCacheNameMatchesWhatAFetchActuallyWrites(t *testing.T) {
|
||||||
|
// The control plane proposes deletions by recomputing this name from the
|
||||||
|
// catalog. If it ever drifted from what a fetch writes, the proposal would
|
||||||
|
// name a file that is in use — so the two must be one function, and this
|
||||||
|
// is what says so.
|
||||||
|
f := &Fetcher{CacheDir: "/cache"}
|
||||||
|
const source = "https://models.internal/x.gguf"
|
||||||
|
|
||||||
|
got, err := f.cachePath(source)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if want := CachePathFor("/cache", source); got != want {
|
||||||
|
t.Errorf("a fetch writes %q but the catalog would look for %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOneOfTwoLookalikeFilesIsNotMistakenForTheOther(t *testing.T) {
|
||||||
|
// That the two cache to different names is covered where the naming lives.
|
||||||
|
// What matters here is the consequence: deleting the unreferenced one must
|
||||||
|
// not take the referenced one with it, and both end in "model.gguf".
|
||||||
|
a := CachePathFor("/cache", "https://one.example/model.gguf")
|
||||||
|
b := CachePathFor("/cache", "https://two.example/model.gguf")
|
||||||
|
|
||||||
|
got := Orphans([]string{a, b}, []string{"https://one.example/model.gguf"})
|
||||||
|
if !slices.Equal(got, []string{b}) {
|
||||||
|
t.Errorf("orphans = %v, want only %s", got, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestASourceThatIsNeverFetchedProtectsNothingAndDeletesNothing(t *testing.T) {
|
||||||
|
// A plain path, or a model id an engine resolves itself, puts nothing in
|
||||||
|
// the cache. Computing a cache name for it would reserve a file that does
|
||||||
|
// not exist, and it must not stop a real orphan being found either.
|
||||||
|
const dir = "/models"
|
||||||
|
orphan := CachePathFor(dir, "https://models.internal/gone.gguf")
|
||||||
|
|
||||||
|
got := Orphans([]string{orphan, filepath.Join(dir, "local.gguf")},
|
||||||
|
[]string{"/models/local.gguf", "Qwen/Qwen2.5-7B-Instruct"})
|
||||||
|
|
||||||
|
if !slices.Equal(got, []string{orphan}) {
|
||||||
|
t.Errorf("orphans = %v, want only %s", got, orphan)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnEmptyCatalogDoesNotMakeOperatorFilesDisposable(t *testing.T) {
|
||||||
|
// The dangerous case: every model removed from the catalog at once. Files
|
||||||
|
// llmgw fetched become orphans, which is right. Files it did not must stay
|
||||||
|
// untouched, which is the whole point.
|
||||||
|
const dir = "/models"
|
||||||
|
have := []string{
|
||||||
|
filepath.Join(dir, "hand-placed.gguf"),
|
||||||
|
CachePathFor(dir, "https://models.internal/a.gguf"),
|
||||||
|
}
|
||||||
|
got := Orphans(have, nil)
|
||||||
|
if len(got) != 1 || got[0] != CachePathFor(dir, "https://models.internal/a.gguf") {
|
||||||
|
t.Errorf("orphans = %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPartialsAreNotSweptUpWithTheRest(t *testing.T) {
|
||||||
|
// A fetch may be running right now, and only the node itself can tell a
|
||||||
|
// live download from one a killed agent left behind.
|
||||||
|
if !PartialName("/models/abc-model.gguf.part") {
|
||||||
|
t.Error("a .part file was not recognised")
|
||||||
|
}
|
||||||
|
if PartialName("/models/abc-model.gguf") {
|
||||||
|
t.Error("a finished file was called partial")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,267 @@
|
|||||||
|
// Package auth authenticates client requests against the key store (plan §7.9).
|
||||||
|
//
|
||||||
|
// The rule that shapes everything here: authentication runs on every request,
|
||||||
|
// and the request path does not touch the database. A key is looked up once,
|
||||||
|
// resolved against its tenancy chain once, and then answered from memory until
|
||||||
|
// the entry expires or is invalidated.
|
||||||
|
package auth
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Principal is who a request is, once authenticated.
|
||||||
|
type Principal struct {
|
||||||
|
KeyID string
|
||||||
|
KeyPrefix string
|
||||||
|
ProjectID string
|
||||||
|
TeamID string
|
||||||
|
OrgID string
|
||||||
|
|
||||||
|
// Band and RoutingProfile come from the key, and are authoritative: with
|
||||||
|
// authentication on, a client can no longer promote itself with a header.
|
||||||
|
Band string
|
||||||
|
RoutingProfile string
|
||||||
|
|
||||||
|
Limits store.Effective
|
||||||
|
}
|
||||||
|
|
||||||
|
// Allows reports whether this principal may use a model.
|
||||||
|
func (p Principal) Allows(model string) bool { return p.Limits.Allows(model) }
|
||||||
|
|
||||||
|
type contextKey struct{}
|
||||||
|
|
||||||
|
// WithPrincipal attaches an authenticated identity to a context.
|
||||||
|
func WithPrincipal(ctx context.Context, p Principal) context.Context {
|
||||||
|
return context.WithValue(ctx, contextKey{}, p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// FromContext returns the authenticated identity, if there is one.
|
||||||
|
//
|
||||||
|
// Absent means authentication is off, which is a deployment choice rather than
|
||||||
|
// a failure: a gateway on a trusted network with no keys configured still
|
||||||
|
// serves. Callers must therefore treat the missing case as "unrestricted", and
|
||||||
|
// the middleware is what guarantees it cannot mean "unchecked".
|
||||||
|
func FromContext(ctx context.Context) (Principal, bool) {
|
||||||
|
p, ok := ctx.Value(contextKey{}).(Principal)
|
||||||
|
return p, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config tunes the authenticator.
|
||||||
|
type Config struct {
|
||||||
|
// TTL is how long a resolved key is trusted before being read again. It
|
||||||
|
// bounds how stale a revocation can be, so it trades a database read per
|
||||||
|
// key per TTL against how long a revoked key keeps working.
|
||||||
|
TTL time.Duration
|
||||||
|
// NegativeTTL is how long a rejected key is remembered as rejected, so a
|
||||||
|
// client looping with a bad key cannot turn into a database read per
|
||||||
|
// request.
|
||||||
|
NegativeTTL time.Duration
|
||||||
|
// TouchInterval is how often a key's last-used time is written, at most.
|
||||||
|
TouchInterval time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultConfig returns sensible values.
|
||||||
|
func DefaultConfig() Config {
|
||||||
|
return Config{TTL: 30 * time.Second, NegativeTTL: 5 * time.Second, TouchInterval: time.Minute}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) normalize() {
|
||||||
|
d := DefaultConfig()
|
||||||
|
if c.TTL <= 0 {
|
||||||
|
c.TTL = d.TTL
|
||||||
|
}
|
||||||
|
if c.NegativeTTL <= 0 {
|
||||||
|
c.NegativeTTL = d.NegativeTTL
|
||||||
|
}
|
||||||
|
if c.TouchInterval <= 0 {
|
||||||
|
c.TouchInterval = d.TouchInterval
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Errors an authenticator returns. They are distinguished because the status
|
||||||
|
// codes differ: a missing credential is 401, a forbidden model is 403.
|
||||||
|
var (
|
||||||
|
ErrMissing = errors.New("no api key")
|
||||||
|
ErrInvalid = errors.New("invalid api key")
|
||||||
|
ErrExpired = errors.New("api key expired")
|
||||||
|
ErrRevoked = errors.New("api key revoked")
|
||||||
|
)
|
||||||
|
|
||||||
|
// Authenticator resolves keys and caches the result.
|
||||||
|
type Authenticator struct {
|
||||||
|
store store.Store
|
||||||
|
cfg Config
|
||||||
|
now func() time.Time
|
||||||
|
|
||||||
|
mu sync.RWMutex
|
||||||
|
cache map[string]entry
|
||||||
|
|
||||||
|
// touch batches last-used timestamps; nothing reads the field in real time
|
||||||
|
// and writing it per request would put a database write back on the path.
|
||||||
|
touchMu sync.Mutex
|
||||||
|
touch map[string]time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
type entry struct {
|
||||||
|
principal Principal
|
||||||
|
err error
|
||||||
|
expires time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// New builds an authenticator over a store.
|
||||||
|
func New(s store.Store, cfg Config) *Authenticator {
|
||||||
|
cfg.normalize()
|
||||||
|
return &Authenticator{
|
||||||
|
store: s, cfg: cfg, now: time.Now,
|
||||||
|
cache: map[string]entry{},
|
||||||
|
touch: map[string]time.Time{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Authenticate resolves a presented credential.
|
||||||
|
func (a *Authenticator) Authenticate(ctx context.Context, presented string) (Principal, error) {
|
||||||
|
if presented == "" {
|
||||||
|
return Principal{}, ErrMissing
|
||||||
|
}
|
||||||
|
// Shape first: a bearer token from another system is rejected without a
|
||||||
|
// lookup, and never reaches the store.
|
||||||
|
if err := store.ParseKey(presented); err != nil {
|
||||||
|
return Principal{}, ErrInvalid
|
||||||
|
}
|
||||||
|
|
||||||
|
hash := store.HashKey(presented)
|
||||||
|
id := string(hash)
|
||||||
|
|
||||||
|
if e, ok := a.lookup(id); ok {
|
||||||
|
if e.err == nil {
|
||||||
|
a.markUsed(e.principal.KeyID)
|
||||||
|
}
|
||||||
|
return e.principal, e.err
|
||||||
|
}
|
||||||
|
|
||||||
|
p, err := a.resolve(ctx, hash)
|
||||||
|
if err != nil && !isRejection(err) {
|
||||||
|
// A database problem is not an authentication answer, so it must not be
|
||||||
|
// cached as one. Failing the request is right — the alternative is
|
||||||
|
// serving traffic whose authorization is unknown.
|
||||||
|
return Principal{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
ttl := a.cfg.TTL
|
||||||
|
if err != nil {
|
||||||
|
ttl = a.cfg.NegativeTTL
|
||||||
|
}
|
||||||
|
a.store_(id, entry{principal: p, err: err, expires: a.now().Add(ttl)})
|
||||||
|
if err == nil {
|
||||||
|
a.markUsed(p.KeyID)
|
||||||
|
}
|
||||||
|
return p, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Authenticator) resolve(ctx context.Context, hash []byte) (Principal, error) {
|
||||||
|
k, err := a.store.KeyByHash(ctx, hash)
|
||||||
|
if errors.Is(err, store.ErrNotFound) {
|
||||||
|
return Principal{}, ErrInvalid
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return Principal{}, err
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case !k.RevokedAt.IsZero():
|
||||||
|
return Principal{}, ErrRevoked
|
||||||
|
case !k.ExpiresAt.IsZero() && !a.now().Before(k.ExpiresAt):
|
||||||
|
return Principal{}, ErrExpired
|
||||||
|
}
|
||||||
|
|
||||||
|
eff, err := a.store.Resolve(ctx, k.ID)
|
||||||
|
if err != nil {
|
||||||
|
return Principal{}, err
|
||||||
|
}
|
||||||
|
return Principal{
|
||||||
|
KeyID: k.ID, KeyPrefix: k.Prefix, ProjectID: eff.ProjectID,
|
||||||
|
TeamID: eff.TeamID, OrgID: eff.OrgID,
|
||||||
|
Band: eff.Band, RoutingProfile: eff.RoutingProfile,
|
||||||
|
Limits: eff,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Authenticator) lookup(id string) (entry, bool) {
|
||||||
|
a.mu.RLock()
|
||||||
|
e, ok := a.cache[id]
|
||||||
|
a.mu.RUnlock()
|
||||||
|
if !ok || a.now().After(e.expires) {
|
||||||
|
return entry{}, false
|
||||||
|
}
|
||||||
|
return e, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Authenticator) store_(id string, e entry) {
|
||||||
|
a.mu.Lock()
|
||||||
|
a.cache[id] = e
|
||||||
|
a.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Invalidate drops every cached decision.
|
||||||
|
//
|
||||||
|
// Coarse on purpose: a limit change anywhere in the tree can change any key
|
||||||
|
// below it, and working out which keys those are costs more than re-resolving
|
||||||
|
// the few that are actually in use.
|
||||||
|
func (a *Authenticator) Invalidate() {
|
||||||
|
a.mu.Lock()
|
||||||
|
a.cache = map[string]entry{}
|
||||||
|
a.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Authenticator) markUsed(keyID string) {
|
||||||
|
if keyID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := a.now()
|
||||||
|
a.touchMu.Lock()
|
||||||
|
if last, ok := a.touch[keyID]; !ok || now.Sub(last) >= a.cfg.TouchInterval {
|
||||||
|
a.touch[keyID] = now
|
||||||
|
}
|
||||||
|
a.touchMu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// FlushTouches writes the pending last-used timestamps. The batch writer calls
|
||||||
|
// it on its own schedule.
|
||||||
|
func (a *Authenticator) FlushTouches(ctx context.Context) error {
|
||||||
|
a.touchMu.Lock()
|
||||||
|
pending := a.touch
|
||||||
|
a.touch = map[string]time.Time{}
|
||||||
|
a.touchMu.Unlock()
|
||||||
|
|
||||||
|
if len(pending) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return a.store.TouchKeys(ctx, pending)
|
||||||
|
}
|
||||||
|
|
||||||
|
func isRejection(err error) bool {
|
||||||
|
return errors.Is(err, ErrInvalid) || errors.Is(err, ErrExpired) ||
|
||||||
|
errors.Is(err, ErrRevoked) || errors.Is(err, ErrMissing)
|
||||||
|
}
|
||||||
|
|
||||||
|
// BearerToken extracts the credential from a request.
|
||||||
|
//
|
||||||
|
// Both spellings are accepted because both are in the wild: the OpenAI SDKs
|
||||||
|
// send Authorization, and some tools only manage to set an api-key header.
|
||||||
|
func BearerToken(r *http.Request) string {
|
||||||
|
if h := r.Header.Get("Authorization"); h != "" {
|
||||||
|
if token, ok := strings.CutPrefix(h, "Bearer "); ok {
|
||||||
|
return strings.TrimSpace(token)
|
||||||
|
}
|
||||||
|
// A bare value is common enough in hand-written clients to accept.
|
||||||
|
return strings.TrimSpace(h)
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(r.Header.Get("api-key"))
|
||||||
|
}
|
||||||
@@ -0,0 +1,440 @@
|
|||||||
|
package auth
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fixture builds a store with one org/team/project and one key.
|
||||||
|
func fixture(t *testing.T) (*store.SQLite, string, string) {
|
||||||
|
t.Helper()
|
||||||
|
ctx := context.Background()
|
||||||
|
s, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "t.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = s.Close() })
|
||||||
|
|
||||||
|
must(t, s.PutOrg(ctx, store.Org{ID: "org1", Name: "Acme"}))
|
||||||
|
must(t, s.PutTeam(ctx, store.Team{ID: "team1", OrgID: "org1", Name: "R"}))
|
||||||
|
must(t, s.PutProject(ctx, store.Project{ID: "proj1", TeamID: "team1", Name: "P"}))
|
||||||
|
|
||||||
|
secret, k, err := store.NewKey("key1", "laptop", "proj1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
k.CreatedAt = time.Now()
|
||||||
|
k.Band = "gold"
|
||||||
|
must(t, s.CreateKey(ctx, k))
|
||||||
|
return s, secret, k.ID
|
||||||
|
}
|
||||||
|
|
||||||
|
func must(t *testing.T, err error) {
|
||||||
|
t.Helper()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAValidKeyResolvesItsWholeChain(t *testing.T) {
|
||||||
|
s, secret, keyID := fixture(t)
|
||||||
|
a := New(s, DefaultConfig())
|
||||||
|
|
||||||
|
p, err := a.Authenticate(context.Background(), secret)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p.KeyID != keyID || p.ProjectID != "proj1" || p.TeamID != "team1" || p.OrgID != "org1" {
|
||||||
|
t.Errorf("principal %+v", p)
|
||||||
|
}
|
||||||
|
if p.Band != "gold" {
|
||||||
|
t.Errorf("band %q", p.Band)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRejections(t *testing.T) {
|
||||||
|
s, secret, keyID := fixture(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
a := New(s, DefaultConfig())
|
||||||
|
|
||||||
|
if _, err := a.Authenticate(ctx, ""); !errors.Is(err, ErrMissing) {
|
||||||
|
t.Errorf("no credential: %v", err)
|
||||||
|
}
|
||||||
|
// A bearer token from some other system must be rejected on shape alone.
|
||||||
|
if _, err := a.Authenticate(ctx, "ghp_0123456789abcdefghij"); !errors.Is(err, ErrInvalid) {
|
||||||
|
t.Errorf("foreign token: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := a.Authenticate(ctx, alter(secret)); !errors.Is(err, ErrInvalid) {
|
||||||
|
t.Errorf("wrong key: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
must(t, s.RevokeKey(ctx, keyID, time.Now()))
|
||||||
|
a.Invalidate()
|
||||||
|
if _, err := a.Authenticate(ctx, secret); !errors.Is(err, ErrRevoked) {
|
||||||
|
t.Errorf("revoked key: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// countingStore counts how often the database is actually read.
|
||||||
|
type countingStore struct {
|
||||||
|
store.Store
|
||||||
|
lookups atomic.Int64
|
||||||
|
fail atomic.Bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *countingStore) KeyByHash(ctx context.Context, hash []byte) (store.Key, error) {
|
||||||
|
c.lookups.Add(1)
|
||||||
|
if c.fail.Load() {
|
||||||
|
return store.Key{}, errors.New("database is down")
|
||||||
|
}
|
||||||
|
return c.Store.KeyByHash(ctx, hash)
|
||||||
|
}
|
||||||
|
|
||||||
|
// alter changes a key by one character, guaranteeing the result differs.
|
||||||
|
//
|
||||||
|
// Replacing the last character with a fixed one does not: keys are base62, so
|
||||||
|
// one in sixty-two already ends in that character and the "wrong" key is the
|
||||||
|
// right one. That test passed locally for a day and failed in CI, which is
|
||||||
|
// exactly how long a one-in-sixty-two flake takes to look like a real bug.
|
||||||
|
func alter(secret string) string {
|
||||||
|
last := secret[len(secret)-1]
|
||||||
|
replacement := byte('a')
|
||||||
|
if last == replacement {
|
||||||
|
replacement = 'b'
|
||||||
|
}
|
||||||
|
return secret[:len(secret)-1] + string(replacement)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheRequestPathDoesNotReadTheDatabase(t *testing.T) {
|
||||||
|
// The rule the whole package is shaped around. Without the cache this is a
|
||||||
|
// database read per request, on every request.
|
||||||
|
s, secret, _ := fixture(t)
|
||||||
|
c := &countingStore{Store: s}
|
||||||
|
a := New(c, DefaultConfig())
|
||||||
|
|
||||||
|
for range 100 {
|
||||||
|
if _, err := a.Authenticate(context.Background(), secret); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n := c.lookups.Load(); n != 1 {
|
||||||
|
t.Errorf("100 requests caused %d database reads", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARejectedKeyIsNotADatabaseReadPerRequest(t *testing.T) {
|
||||||
|
// A client looping with a bad key would otherwise be a free way to load the
|
||||||
|
// database from outside.
|
||||||
|
s, _, _ := fixture(t)
|
||||||
|
c := &countingStore{Store: s}
|
||||||
|
a := New(c, DefaultConfig())
|
||||||
|
|
||||||
|
bad := store.KeyPrefix + "0000000000000000000000000000000000000000000"
|
||||||
|
for range 50 {
|
||||||
|
if _, err := a.Authenticate(context.Background(), bad); !errors.Is(err, ErrInvalid) {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n := c.lookups.Load(); n != 1 {
|
||||||
|
t.Errorf("50 bad requests caused %d database reads", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCachedDecisionsExpire(t *testing.T) {
|
||||||
|
// The cache bounds how long a revocation can take to take effect, so it has
|
||||||
|
// to actually expire.
|
||||||
|
s, secret, keyID := fixture(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
a := New(s, Config{TTL: time.Second, NegativeTTL: time.Second})
|
||||||
|
a.now = func() time.Time { return now }
|
||||||
|
|
||||||
|
if _, err := a.Authenticate(ctx, secret); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
must(t, s.RevokeKey(ctx, keyID, now))
|
||||||
|
|
||||||
|
// Still inside the TTL: the stale answer is the point of the cache.
|
||||||
|
if _, err := a.Authenticate(ctx, secret); err != nil {
|
||||||
|
t.Errorf("the cache expired early: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
now = now.Add(2 * time.Second)
|
||||||
|
if _, err := a.Authenticate(ctx, secret); !errors.Is(err, ErrRevoked) {
|
||||||
|
t.Errorf("a revoked key still works after the TTL: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAStoreFailureIsNotAnAuthenticationAnswer(t *testing.T) {
|
||||||
|
// Two things must not happen: the failure must not be cached as a
|
||||||
|
// rejection, and it must not read to the caller as a bad key.
|
||||||
|
s, secret, _ := fixture(t)
|
||||||
|
c := &countingStore{Store: s}
|
||||||
|
a := New(c, DefaultConfig())
|
||||||
|
|
||||||
|
c.fail.Store(true)
|
||||||
|
_, err := a.Authenticate(context.Background(), secret)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a broken database authenticated a request")
|
||||||
|
}
|
||||||
|
if isRejection(err) {
|
||||||
|
t.Errorf("a database failure was reported as a credential problem: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
c.fail.Store(false)
|
||||||
|
if _, err := a.Authenticate(context.Background(), secret); err != nil {
|
||||||
|
t.Errorf("the failure was cached: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExpiryIsNotCached(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
s, _, _ := fixture(t)
|
||||||
|
|
||||||
|
secret, k, _ := store.NewKey("key2", "temp", "proj1")
|
||||||
|
k.CreatedAt = time.Now()
|
||||||
|
k.ExpiresAt = time.Now().Add(time.Hour)
|
||||||
|
must(t, s.CreateKey(ctx, k))
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
a := New(s, Config{TTL: time.Minute})
|
||||||
|
a.now = func() time.Time { return now }
|
||||||
|
|
||||||
|
if _, err := a.Authenticate(ctx, secret); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Past the key's expiry but inside the cache TTL. The cached decision is
|
||||||
|
// allowed to be stale by at most the TTL, which is the documented trade.
|
||||||
|
now = now.Add(2 * time.Hour)
|
||||||
|
if _, err := a.Authenticate(ctx, secret); !errors.Is(err, ErrExpired) {
|
||||||
|
t.Errorf("an expired key authenticated: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConcurrentAuthentication(t *testing.T) {
|
||||||
|
s, secret, _ := fixture(t)
|
||||||
|
a := New(s, DefaultConfig())
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
errs := make(chan error, 64)
|
||||||
|
for range 64 {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
if _, err := a.Authenticate(context.Background(), secret); err != nil {
|
||||||
|
errs <- err
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
close(errs)
|
||||||
|
for err := range errs {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMiddlewareGuardsEveryRouteButTheExemptOnes(t *testing.T) {
|
||||||
|
s, secret, _ := fixture(t)
|
||||||
|
a := New(s, DefaultConfig())
|
||||||
|
|
||||||
|
var reached atomic.Int64
|
||||||
|
inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
reached.Add(1)
|
||||||
|
if _, ok := FromContext(r.Context()); !ok && r.URL.Path != "/health" {
|
||||||
|
t.Error("an authenticated request carried no principal")
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
})
|
||||||
|
srv := httptest.NewServer(Middleware(a, nil, "/health")(inner))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
get := func(path, token string) int {
|
||||||
|
req, _ := http.NewRequest(http.MethodGet, srv.URL+path, nil) //nolint:noctx // test
|
||||||
|
if token != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+token)
|
||||||
|
}
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
return resp.StatusCode
|
||||||
|
}
|
||||||
|
|
||||||
|
if code := get("/health", ""); code != http.StatusOK {
|
||||||
|
t.Errorf("health needs no key, got %d", code)
|
||||||
|
}
|
||||||
|
if code := get("/v1/models", ""); code != http.StatusUnauthorized {
|
||||||
|
t.Errorf("an unauthenticated request got %d", code)
|
||||||
|
}
|
||||||
|
if code := get("/v1/models", "sk-llmgw-bogus"); code != http.StatusUnauthorized {
|
||||||
|
t.Errorf("a bad key got %d", code)
|
||||||
|
}
|
||||||
|
if code := get("/v1/models", secret); code != http.StatusOK {
|
||||||
|
t.Errorf("a good key got %d", code)
|
||||||
|
}
|
||||||
|
if n := reached.Load(); n != 2 {
|
||||||
|
t.Errorf("the handler was reached %d times; only health and the valid key should pass", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARevokedKeyLooksExactlyLikeAnUnknownOne(t *testing.T) {
|
||||||
|
// Telling a caller that a key exists but was revoked confirms the key was
|
||||||
|
// real, which is information worth withholding.
|
||||||
|
s, secret, keyID := fixture(t)
|
||||||
|
must(t, s.RevokeKey(context.Background(), keyID, time.Now()))
|
||||||
|
|
||||||
|
a := New(s, DefaultConfig())
|
||||||
|
srv := httptest.NewServer(Middleware(a, nil)(http.HandlerFunc(
|
||||||
|
func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
body := func(token string) string {
|
||||||
|
req, _ := http.NewRequest(http.MethodGet, srv.URL, nil) //nolint:noctx // test
|
||||||
|
req.Header.Set("Authorization", "Bearer "+token)
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
var e struct {
|
||||||
|
Error struct{ Message string } `json:"error"`
|
||||||
|
}
|
||||||
|
_ = json.NewDecoder(resp.Body).Decode(&e)
|
||||||
|
return e.Error.Message
|
||||||
|
}
|
||||||
|
|
||||||
|
unknown := store.KeyPrefix + "0000000000000000000000000000000000000000000"
|
||||||
|
if body(secret) != body(unknown) {
|
||||||
|
t.Errorf("revoked says %q, unknown says %q", body(secret), body(unknown))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAStoreFailureDoesNotReadAsABadKey(t *testing.T) {
|
||||||
|
s, secret, _ := fixture(t)
|
||||||
|
c := &countingStore{Store: s}
|
||||||
|
c.fail.Store(true)
|
||||||
|
a := New(c, DefaultConfig())
|
||||||
|
|
||||||
|
srv := httptest.NewServer(Middleware(a, nil)(http.HandlerFunc(
|
||||||
|
func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
req, _ := http.NewRequest(http.MethodGet, srv.URL, nil) //nolint:noctx // test
|
||||||
|
req.Header.Set("Authorization", "Bearer "+secret)
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
// 503 rather than 401: sending someone hunting for a key problem that does
|
||||||
|
// not exist wastes the one thing an incident is short of.
|
||||||
|
if resp.StatusCode != http.StatusServiceUnavailable {
|
||||||
|
t.Errorf("status %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBearerTokenAcceptsWhatClientsActuallySend(t *testing.T) {
|
||||||
|
cases := map[string]string{
|
||||||
|
"Bearer sk-llmgw-abc": "sk-llmgw-abc",
|
||||||
|
"sk-llmgw-abc": "sk-llmgw-abc",
|
||||||
|
"Bearer sk-llmgw-abc": "sk-llmgw-abc",
|
||||||
|
"": "",
|
||||||
|
}
|
||||||
|
for header, want := range cases {
|
||||||
|
r := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||||
|
if header != "" {
|
||||||
|
r.Header.Set("Authorization", header)
|
||||||
|
}
|
||||||
|
if got := BearerToken(r); got != want {
|
||||||
|
t.Errorf("Authorization %q -> %q, want %q", header, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Some tools only manage to set this one.
|
||||||
|
r := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||||
|
r.Header.Set("api-key", "sk-llmgw-xyz")
|
||||||
|
if got := BearerToken(r); got != "sk-llmgw-xyz" {
|
||||||
|
t.Errorf("api-key header -> %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLastUsedIsBatchedNotWrittenPerRequest(t *testing.T) {
|
||||||
|
s, secret, keyID := fixture(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
a := New(s, Config{TTL: time.Minute, TouchInterval: time.Hour})
|
||||||
|
|
||||||
|
for range 20 {
|
||||||
|
if _, err := a.Authenticate(ctx, secret); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Nothing written yet: the field is not read in real time.
|
||||||
|
k, _ := s.KeyByHash(ctx, store.HashKey(secret))
|
||||||
|
if !k.LastUsedAt.IsZero() {
|
||||||
|
t.Error("last-used was written on the request path")
|
||||||
|
}
|
||||||
|
|
||||||
|
must(t, a.FlushTouches(ctx))
|
||||||
|
k, _ = s.KeyByHash(ctx, store.HashKey(secret))
|
||||||
|
if k.LastUsedAt.IsZero() {
|
||||||
|
t.Error("last-used was never written")
|
||||||
|
}
|
||||||
|
if k.ID != keyID {
|
||||||
|
t.Errorf("wrong key touched: %s", k.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExemptingTheLandingPageDoesNotExemptEverything(t *testing.T) {
|
||||||
|
// A near-miss worth a test. "/" ends in a slash, so a prefix rule would
|
||||||
|
// match every path there is: authentication still installed, still logging,
|
||||||
|
// and letting the entire API through.
|
||||||
|
cases := []struct {
|
||||||
|
path string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"/", true},
|
||||||
|
{"/_static/base.css", true},
|
||||||
|
{"/health", true},
|
||||||
|
{"/v1/models", false},
|
||||||
|
{"/v1/chat/completions", false},
|
||||||
|
{"/healthz", false}, // not a prefix rule
|
||||||
|
{"/_staticx", false}, // prefix rules end at the slash
|
||||||
|
}
|
||||||
|
exempt := []string{"/health", "/", "/_static/"}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := isExempt(c.path, exempt); got != c.want {
|
||||||
|
t.Errorf("isExempt(%q) = %v, want %v", c.path, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAlterAlwaysChangesTheKey(t *testing.T) {
|
||||||
|
// The helper the test above depends on. If it can return its input, the
|
||||||
|
// test it serves silently stops testing anything.
|
||||||
|
for range 500 {
|
||||||
|
secret, _, err := store.NewKey("id", "n", "p")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if alter(secret) == secret {
|
||||||
|
t.Fatalf("alter returned its input for %q", secret)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Including the case that made the original flaky: a key ending in the
|
||||||
|
// replacement character.
|
||||||
|
if got := alter("sk-llmgw-aaaa"); got == "sk-llmgw-aaaa" {
|
||||||
|
t.Error("alter is a no-op when the key already ends in the replacement")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
package auth
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Middleware authenticates every request before it reaches the router.
|
||||||
|
//
|
||||||
|
// It is a wrapper rather than a check inside the handler so there is no path
|
||||||
|
// into the API that forgets to call it. Adding a route cannot accidentally add
|
||||||
|
// an unauthenticated one.
|
||||||
|
func Middleware(a *Authenticator, log *slog.Logger, exempt ...string) func(http.Handler) http.Handler {
|
||||||
|
if log == nil {
|
||||||
|
log = slog.Default()
|
||||||
|
}
|
||||||
|
return func(next http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if isExempt(r.URL.Path, exempt) {
|
||||||
|
next.ServeHTTP(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p, err := a.Authenticate(r.Context(), BearerToken(r))
|
||||||
|
if err != nil {
|
||||||
|
writeAuthError(w, err)
|
||||||
|
// The key prefix, never the key: a rejected credential in a log
|
||||||
|
// is still a credential.
|
||||||
|
log.Debug("rejected a request", "path", r.URL.Path,
|
||||||
|
"reason", err, "remote", r.RemoteAddr)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
next.ServeHTTP(w, r.WithContext(WithPrincipal(r.Context(), p)))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// isExempt reports whether a path is served without a key.
|
||||||
|
//
|
||||||
|
// An entry ending in "/" is a prefix rule, with one exception that matters
|
||||||
|
// enormously: "/" itself is exact. Treating it as a prefix would match every
|
||||||
|
// path there is, so exempting the landing page would silently exempt the whole
|
||||||
|
// API — authentication still installed, still logging, and letting everything
|
||||||
|
// through.
|
||||||
|
//
|
||||||
|
// Metrics and pprof are not in this list because they live on the admin
|
||||||
|
// listener, a separate port precisely so it can be firewalled rather than
|
||||||
|
// authenticated.
|
||||||
|
func isExempt(path string, exempt []string) bool {
|
||||||
|
for _, e := range exempt {
|
||||||
|
if path == e {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if e != "/" && strings.HasSuffix(e, "/") && strings.HasPrefix(path, e) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeAuthError(w http.ResponseWriter, err error) {
|
||||||
|
status, kind, msg := http.StatusUnauthorized, "invalid_request_error", "invalid api key"
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, ErrMissing):
|
||||||
|
msg = "no api key was provided"
|
||||||
|
case errors.Is(err, ErrExpired):
|
||||||
|
msg = "this api key has expired"
|
||||||
|
case errors.Is(err, ErrRevoked):
|
||||||
|
// Deliberately the same message as an unknown key. Telling a caller
|
||||||
|
// that a key exists but was revoked confirms the key was real.
|
||||||
|
msg = "invalid api key"
|
||||||
|
case errors.Is(err, ErrInvalid):
|
||||||
|
default:
|
||||||
|
// A store failure is ours, not the caller's, and must not read as a
|
||||||
|
// rejected credential — that would send someone hunting for a key
|
||||||
|
// problem that does not exist.
|
||||||
|
status, kind, msg = http.StatusServiceUnavailable, "server_error",
|
||||||
|
"authentication is temporarily unavailable"
|
||||||
|
}
|
||||||
|
|
||||||
|
if status == http.StatusUnauthorized {
|
||||||
|
// What the OpenAI SDKs look for to stop retrying.
|
||||||
|
w.Header().Set("WWW-Authenticate", `Bearer realm="llmgw"`)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(status)
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{
|
||||||
|
"message": msg, "type": kind, "code": nil,
|
||||||
|
}})
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteForbiddenModel reports a model this key may not use.
|
||||||
|
//
|
||||||
|
// Separate from authentication because it happens later: the model is only
|
||||||
|
// known once the body has been parsed.
|
||||||
|
func WriteForbiddenModel(w http.ResponseWriter, model string) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(http.StatusForbidden)
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{
|
||||||
|
"message": "this api key is not permitted to use model " + model,
|
||||||
|
"type": "invalid_request_error", "code": nil,
|
||||||
|
}})
|
||||||
|
}
|
||||||
@@ -0,0 +1,437 @@
|
|||||||
|
// Package bench measures what a replica can actually do, so the numbers the
|
||||||
|
// router depends on stop being guesses (plan §6.2.1).
|
||||||
|
//
|
||||||
|
// Three of them matter: the weight a replica gets when the scheduler compares
|
||||||
|
// it to others, the queue depth admission treats as healthy, and the KV usage
|
||||||
|
// it treats as full. All three are currently hand-written in configuration, and
|
||||||
|
// a hand-written capacity number is a hand-written opinion about hardware
|
||||||
|
// nobody measured.
|
||||||
|
//
|
||||||
|
// The measurement is a load ramp. Concurrency climbs until the SLO breaks, and
|
||||||
|
// the last level that still met it is the capacity — not peak throughput, which
|
||||||
|
// a GPU will happily deliver long after latency has become unusable.
|
||||||
|
package bench
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"math/rand/v2"
|
||||||
|
"net/http"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SLO is the latency target capacity is measured against.
|
||||||
|
type SLO struct {
|
||||||
|
TTFTp99 time.Duration
|
||||||
|
TPOTp99 time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// Met reports whether a step's latencies satisfy the target.
|
||||||
|
func (s SLO) Met(ttft, tpot time.Duration) bool {
|
||||||
|
if s.TTFTp99 > 0 && ttft > s.TTFTp99 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if s.TPOTp99 > 0 && tpot > s.TPOTp99 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config describes a run.
|
||||||
|
type Config struct {
|
||||||
|
// Target is the base URL: the gateway, or one replica directly. Pointing at
|
||||||
|
// a replica measures that replica; pointing at the gateway measures the
|
||||||
|
// whole path including scheduling, which is a different question.
|
||||||
|
Target string
|
||||||
|
// MetricsURL is an engine's /metrics, sampled during the run so queue depth
|
||||||
|
// and KV usage can be reported alongside latency. Only meaningful when
|
||||||
|
// Target is a single replica.
|
||||||
|
MetricsURL string
|
||||||
|
// Engine names which metric vocabulary MetricsURL speaks. Getting it wrong
|
||||||
|
// is silent: nothing matches, every sample reads zero, and the suggested
|
||||||
|
// thresholds are built on nothing.
|
||||||
|
Engine string
|
||||||
|
Model string
|
||||||
|
APIKey string
|
||||||
|
|
||||||
|
// Concurrency is the ramp. Each level runs for Duration.
|
||||||
|
Concurrency []int
|
||||||
|
Duration time.Duration
|
||||||
|
|
||||||
|
PromptTokens int
|
||||||
|
MaxTokens int
|
||||||
|
|
||||||
|
// SharedPrefix sends every request the same prompt.
|
||||||
|
//
|
||||||
|
// Off by default, and that default is the whole point: with a shared prefix
|
||||||
|
// the engine's cache answers most of the prefill and the measurement
|
||||||
|
// describes the cache rather than the hardware. Turning it on measures the
|
||||||
|
// cached case deliberately, which is a fair question to ask separately.
|
||||||
|
SharedPrefix bool
|
||||||
|
|
||||||
|
SLO SLO
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultConfig returns a ramp that covers most single-replica hardware.
|
||||||
|
func DefaultConfig() Config {
|
||||||
|
return Config{
|
||||||
|
Concurrency: []int{1, 2, 4, 8, 16, 32, 64, 128},
|
||||||
|
Duration: 20 * time.Second,
|
||||||
|
PromptTokens: 512,
|
||||||
|
MaxTokens: 128,
|
||||||
|
SLO: SLO{TTFTp99: 2 * time.Second, TPOTp99: 50 * time.Millisecond},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) normalize() error {
|
||||||
|
d := DefaultConfig()
|
||||||
|
if c.Target == "" {
|
||||||
|
return errors.New("bench: a target is required")
|
||||||
|
}
|
||||||
|
if c.Model == "" {
|
||||||
|
return errors.New("bench: a model is required")
|
||||||
|
}
|
||||||
|
c.Target = strings.TrimRight(c.Target, "/")
|
||||||
|
if len(c.Concurrency) == 0 {
|
||||||
|
c.Concurrency = d.Concurrency
|
||||||
|
}
|
||||||
|
if c.Duration <= 0 {
|
||||||
|
c.Duration = d.Duration
|
||||||
|
}
|
||||||
|
if c.PromptTokens <= 0 {
|
||||||
|
c.PromptTokens = d.PromptTokens
|
||||||
|
}
|
||||||
|
if c.MaxTokens <= 0 {
|
||||||
|
c.MaxTokens = d.MaxTokens
|
||||||
|
}
|
||||||
|
sort.Ints(c.Concurrency)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Percentiles summarizes a latency sample.
|
||||||
|
type Percentiles struct {
|
||||||
|
P50, P90, P99, Max time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step is one concurrency level's result.
|
||||||
|
type Step struct {
|
||||||
|
Concurrency int
|
||||||
|
Completed int
|
||||||
|
Failed int
|
||||||
|
// Requests and output tokens per second, over the step's wall clock.
|
||||||
|
RequestsPerSec float64
|
||||||
|
TokensPerSec float64
|
||||||
|
TTFT Percentiles
|
||||||
|
// TPOT is time per output token after the first, which is what a reader
|
||||||
|
// experiences as the speed of the stream.
|
||||||
|
TPOT Percentiles
|
||||||
|
Waiting float64
|
||||||
|
Running float64
|
||||||
|
KVUsage float64
|
||||||
|
MeetsSLO bool
|
||||||
|
// FirstError is kept so a step that failed says why rather than just
|
||||||
|
// reporting zeroes.
|
||||||
|
FirstError string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Result is the whole ramp plus what it concluded.
|
||||||
|
type Result struct {
|
||||||
|
Model string
|
||||||
|
Target string
|
||||||
|
Steps []Step
|
||||||
|
SLO SLO
|
||||||
|
RanAt time.Time
|
||||||
|
Elapsed time.Duration
|
||||||
|
|
||||||
|
// MaxConcurrencyAtSLO is the capacity: the highest level that still met the
|
||||||
|
// target. Zero means even one request did not.
|
||||||
|
MaxConcurrencyAtSLO int
|
||||||
|
// ThroughputTokPerSec and the latencies are taken at that level.
|
||||||
|
ThroughputTokPerSec float64
|
||||||
|
TTFTp99, TPOTp99 time.Duration
|
||||||
|
// SuggestedTargetWaiting and SuggestedKVCeiling are what admission should
|
||||||
|
// treat as healthy, observed rather than guessed.
|
||||||
|
SuggestedTargetWaiting int
|
||||||
|
SuggestedKVCeiling float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run executes the ramp.
|
||||||
|
func Run(ctx context.Context, cfg Config, progress func(Step)) (Result, error) {
|
||||||
|
if err := cfg.normalize(); err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
res := Result{Model: cfg.Model, Target: cfg.Target, SLO: cfg.SLO, RanAt: time.Now()}
|
||||||
|
start := time.Now()
|
||||||
|
|
||||||
|
client := &http.Client{Timeout: 10 * time.Minute}
|
||||||
|
for _, c := range cfg.Concurrency {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
step := runStep(ctx, client, cfg, c)
|
||||||
|
res.Steps = append(res.Steps, step)
|
||||||
|
if progress != nil {
|
||||||
|
progress(step)
|
||||||
|
}
|
||||||
|
// Stop climbing once the target is missed. Everything above is slower
|
||||||
|
// by construction, and the cost of finding out is real GPU time.
|
||||||
|
if !step.MeetsSLO {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
res.Elapsed = time.Since(start)
|
||||||
|
res.conclude()
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Result) conclude() {
|
||||||
|
for _, s := range r.Steps {
|
||||||
|
if !s.MeetsSLO || s.Completed == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if s.Concurrency <= r.MaxConcurrencyAtSLO {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
r.MaxConcurrencyAtSLO = s.Concurrency
|
||||||
|
r.ThroughputTokPerSec = s.TokensPerSec
|
||||||
|
r.TTFTp99, r.TPOTp99 = s.TTFT.P99, s.TPOT.P99
|
||||||
|
// The queue depth and cache pressure seen while still meeting the SLO
|
||||||
|
// are exactly what admission should treat as healthy.
|
||||||
|
r.SuggestedTargetWaiting = int(s.Waiting + 0.5)
|
||||||
|
r.SuggestedKVCeiling = roundTo(s.KVUsage, 0.05)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runStep(ctx context.Context, client *http.Client, cfg Config, concurrency int) Step {
|
||||||
|
step := Step{Concurrency: concurrency}
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, cfg.Duration)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
var (
|
||||||
|
mu sync.Mutex
|
||||||
|
ttfts []time.Duration
|
||||||
|
tpots []time.Duration
|
||||||
|
tokens int64
|
||||||
|
failed int
|
||||||
|
firstErr string
|
||||||
|
)
|
||||||
|
|
||||||
|
sampler := newSampler(cfg.MetricsURL, cfg.Engine)
|
||||||
|
sampler.start(ctx)
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for range concurrency {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
r, err := one(ctx, client, cfg)
|
||||||
|
mu.Lock()
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
// A cancelled context is the step ending, not a failure.
|
||||||
|
if ctx.Err() == nil {
|
||||||
|
failed++
|
||||||
|
if firstErr == "" {
|
||||||
|
firstErr = err.Error()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
ttfts = append(ttfts, r.ttft)
|
||||||
|
if r.tpot > 0 {
|
||||||
|
tpots = append(tpots, r.tpot)
|
||||||
|
}
|
||||||
|
tokens += r.tokens
|
||||||
|
}
|
||||||
|
mu.Unlock()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
elapsed := time.Since(start)
|
||||||
|
sampler.stop()
|
||||||
|
|
||||||
|
step.Completed = len(ttfts)
|
||||||
|
step.Failed = failed
|
||||||
|
step.FirstError = firstErr
|
||||||
|
step.TTFT = percentiles(ttfts)
|
||||||
|
step.TPOT = percentiles(tpots)
|
||||||
|
if elapsed > 0 {
|
||||||
|
step.RequestsPerSec = float64(step.Completed) / elapsed.Seconds()
|
||||||
|
step.TokensPerSec = float64(tokens) / elapsed.Seconds()
|
||||||
|
}
|
||||||
|
step.Waiting, step.Running, step.KVUsage = sampler.averages()
|
||||||
|
|
||||||
|
// A step that could not complete anything has not met anything.
|
||||||
|
step.MeetsSLO = step.Completed > 0 && step.Failed == 0 &&
|
||||||
|
cfg.SLO.Met(step.TTFT.P99, step.TPOT.P99)
|
||||||
|
return step
|
||||||
|
}
|
||||||
|
|
||||||
|
type outcome struct {
|
||||||
|
ttft time.Duration
|
||||||
|
tpot time.Duration
|
||||||
|
tokens int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// one sends a single streaming completion and times it.
|
||||||
|
func one(ctx context.Context, client *http.Client, cfg Config) (outcome, error) {
|
||||||
|
body, err := json.Marshal(map[string]any{
|
||||||
|
"model": cfg.Model,
|
||||||
|
"stream": true,
|
||||||
|
"max_tokens": cfg.MaxTokens,
|
||||||
|
"messages": []map[string]string{
|
||||||
|
{"role": "user", "content": prompt(cfg)},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return outcome{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||||
|
cfg.Target+"/v1/chat/completions", bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return outcome{}, err
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
if cfg.APIKey != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+cfg.APIKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return outcome{}, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||||
|
return outcome{}, fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(b)))
|
||||||
|
}
|
||||||
|
|
||||||
|
var out outcome
|
||||||
|
var firstAt, lastAt time.Time
|
||||||
|
sc := bufio.NewScanner(resp.Body)
|
||||||
|
sc.Buffer(make([]byte, 0, 64*1024), 1<<20)
|
||||||
|
for sc.Scan() {
|
||||||
|
line := sc.Text()
|
||||||
|
if !strings.HasPrefix(line, "data: ") || strings.HasSuffix(line, "[DONE]") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Only chunks that carry content count: a role-only first chunk is not
|
||||||
|
// a token, and counting it would flatter the time to first token.
|
||||||
|
if !hasContent(line[6:]) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
if out.tokens == 0 {
|
||||||
|
firstAt = now
|
||||||
|
out.ttft = now.Sub(start)
|
||||||
|
}
|
||||||
|
lastAt = now
|
||||||
|
out.tokens++
|
||||||
|
}
|
||||||
|
if err := sc.Err(); err != nil {
|
||||||
|
return outcome{}, err
|
||||||
|
}
|
||||||
|
if out.tokens == 0 {
|
||||||
|
return outcome{}, errors.New("the response carried no content")
|
||||||
|
}
|
||||||
|
if out.tokens > 1 {
|
||||||
|
out.tpot = lastAt.Sub(firstAt) / time.Duration(out.tokens-1)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasContent(payload string) bool {
|
||||||
|
var chunk struct {
|
||||||
|
Choices []struct {
|
||||||
|
Delta struct {
|
||||||
|
Content string `json:"content"`
|
||||||
|
} `json:"delta"`
|
||||||
|
} `json:"choices"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(payload), &chunk); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, c := range chunk.Choices {
|
||||||
|
if c.Delta.Content != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// prompt builds a request body of roughly the configured size.
|
||||||
|
//
|
||||||
|
// Unique per request unless a shared prefix was asked for: repeating one prompt
|
||||||
|
// lets the engine's cache answer most of the prefill, and the result then
|
||||||
|
// describes the cache rather than the hardware.
|
||||||
|
//
|
||||||
|
// "Shared" has to mean byte-identical. Dropping only the random header while
|
||||||
|
// still picking words at random would leave every prompt different after the
|
||||||
|
// first few tokens, so the cache would miss and the flag would quietly measure
|
||||||
|
// the thing it exists to avoid measuring.
|
||||||
|
func prompt(cfg Config) string {
|
||||||
|
var b strings.Builder
|
||||||
|
if cfg.SharedPrefix {
|
||||||
|
for i := 0; b.Len() < cfg.PromptTokens*4; i++ {
|
||||||
|
b.WriteString(words[i%len(words)])
|
||||||
|
b.WriteByte(' ')
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(&b, "%016x %016x ", rand.Uint64(), rand.Uint64()) //nolint:gosec // not security
|
||||||
|
// Roughly four characters to a token, the same crude ratio the router
|
||||||
|
// uses; exactness is not what this number is for.
|
||||||
|
for b.Len() < cfg.PromptTokens*4 {
|
||||||
|
b.WriteString(words[rand.IntN(len(words))]) //nolint:gosec // not security
|
||||||
|
b.WriteByte(' ')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.WriteString("\n\nSummarize the text above.")
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
var words = strings.Fields(`the quick brown fox jumps over lazy dog while distant
|
||||||
|
thunder rolls across an empty valley and somewhere a door closes softly in the
|
||||||
|
dark house where nobody has lived for years although the garden still produces
|
||||||
|
apples every autumn which fall and rot beneath the trees unnoticed by anyone
|
||||||
|
except the birds that come each morning to pick at them before the frost`)
|
||||||
|
|
||||||
|
func percentiles(v []time.Duration) Percentiles {
|
||||||
|
if len(v) == 0 {
|
||||||
|
return Percentiles{}
|
||||||
|
}
|
||||||
|
s := make([]time.Duration, len(v))
|
||||||
|
copy(s, v)
|
||||||
|
sort.Slice(s, func(i, j int) bool { return s[i] < s[j] })
|
||||||
|
at := func(p float64) time.Duration {
|
||||||
|
i := int(float64(len(s)-1) * p)
|
||||||
|
return s[i]
|
||||||
|
}
|
||||||
|
return Percentiles{P50: at(0.50), P90: at(0.90), P99: at(0.99), Max: s[len(s)-1]}
|
||||||
|
}
|
||||||
|
|
||||||
|
// roundTo rounds to a multiple of step, through integers.
|
||||||
|
//
|
||||||
|
// The obvious float arithmetic returns 0.7000000000000001 for what should be
|
||||||
|
// 0.70, which is harmless on a terminal and ugly in the JSON someone pastes
|
||||||
|
// into a configuration file.
|
||||||
|
func roundTo(v, step float64) float64 {
|
||||||
|
if step <= 0 {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
const scale = 10000
|
||||||
|
units := int64(v/step + 0.5)
|
||||||
|
return float64(int64(float64(units)*step*scale+0.5)) / scale
|
||||||
|
}
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
package bench
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCapacityIsTheLastLevelThatMetTheSLONotThePeak(t *testing.T) {
|
||||||
|
// A GPU keeps delivering more tokens per second long after latency has
|
||||||
|
// become unusable. Taking peak throughput as capacity would tell the router
|
||||||
|
// a replica can serve a load at which nobody would want the answers.
|
||||||
|
r := Result{SLO: SLO{TTFTp99: time.Second}}
|
||||||
|
r.Steps = []Step{
|
||||||
|
{Concurrency: 1, Completed: 10, TokensPerSec: 100, MeetsSLO: true,
|
||||||
|
TTFT: Percentiles{P99: 100 * time.Millisecond}},
|
||||||
|
{Concurrency: 8, Completed: 80, TokensPerSec: 700, MeetsSLO: true,
|
||||||
|
TTFT: Percentiles{P99: 900 * time.Millisecond}, Waiting: 3.4, KVUsage: 0.71},
|
||||||
|
// Faster in tokens, far too slow in latency.
|
||||||
|
{Concurrency: 32, Completed: 200, TokensPerSec: 1500, MeetsSLO: false,
|
||||||
|
TTFT: Percentiles{P99: 9 * time.Second}},
|
||||||
|
}
|
||||||
|
r.conclude()
|
||||||
|
|
||||||
|
if r.MaxConcurrencyAtSLO != 8 {
|
||||||
|
t.Errorf("capacity = %d, want 8", r.MaxConcurrencyAtSLO)
|
||||||
|
}
|
||||||
|
if r.ThroughputTokPerSec != 700 {
|
||||||
|
t.Errorf("throughput = %v; it must come from the level that met the SLO", r.ThroughputTokPerSec)
|
||||||
|
}
|
||||||
|
if r.SuggestedTargetWaiting != 3 {
|
||||||
|
t.Errorf("target_waiting = %d, want 3", r.SuggestedTargetWaiting)
|
||||||
|
}
|
||||||
|
if r.SuggestedKVCeiling != 0.70 {
|
||||||
|
t.Errorf("kv_ceiling = %v, want 0.70", r.SuggestedKVCeiling)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAStepWithFailuresHasNotMetAnything(t *testing.T) {
|
||||||
|
// Latency percentiles over the requests that survived say nothing useful
|
||||||
|
// when the rest were refused.
|
||||||
|
r := Result{SLO: SLO{TTFTp99: time.Second}}
|
||||||
|
r.Steps = []Step{
|
||||||
|
{Concurrency: 4, Completed: 10, Failed: 90, TokensPerSec: 50, MeetsSLO: false,
|
||||||
|
TTFT: Percentiles{P99: 50 * time.Millisecond}},
|
||||||
|
}
|
||||||
|
r.conclude()
|
||||||
|
if r.MaxConcurrencyAtSLO != 0 {
|
||||||
|
t.Errorf("a step that failed 90 of 100 requests was counted as capacity")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSLOIsMissedWhenEitherHalfIs(t *testing.T) {
|
||||||
|
s := SLO{TTFTp99: time.Second, TPOTp99: 50 * time.Millisecond}
|
||||||
|
cases := []struct {
|
||||||
|
ttft, tpot time.Duration
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{500 * time.Millisecond, 20 * time.Millisecond, true},
|
||||||
|
{2 * time.Second, 20 * time.Millisecond, false},
|
||||||
|
{500 * time.Millisecond, 80 * time.Millisecond, false},
|
||||||
|
{2 * time.Second, 80 * time.Millisecond, false},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := s.Met(c.ttft, c.tpot); got != c.want {
|
||||||
|
t.Errorf("Met(%s, %s) = %v", c.ttft, c.tpot, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A target of zero is no target, not a target of zero.
|
||||||
|
none := SLO{}
|
||||||
|
if !none.Met(time.Hour, time.Hour) {
|
||||||
|
t.Error("an unset SLO refused everything")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPercentiles(t *testing.T) {
|
||||||
|
var v []time.Duration
|
||||||
|
for i := 1; i <= 100; i++ {
|
||||||
|
v = append(v, time.Duration(i)*time.Millisecond)
|
||||||
|
}
|
||||||
|
p := percentiles(v)
|
||||||
|
if p.P50 != 50*time.Millisecond || p.P90 != 90*time.Millisecond {
|
||||||
|
t.Errorf("p50=%s p90=%s", p.P50, p.P90)
|
||||||
|
}
|
||||||
|
if p.Max != 100*time.Millisecond {
|
||||||
|
t.Errorf("max=%s", p.Max)
|
||||||
|
}
|
||||||
|
if got := percentiles(nil); got != (Percentiles{}) {
|
||||||
|
t.Errorf("empty sample gave %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptsAreUniqueUnlessAskedNotToBe(t *testing.T) {
|
||||||
|
// The default matters more than it looks. With one repeated prompt the
|
||||||
|
// engine's cache answers most of the prefill, and the measurement describes
|
||||||
|
// the cache rather than the hardware.
|
||||||
|
cfg := Config{PromptTokens: 64}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for range 100 {
|
||||||
|
seen[prompt(cfg)] = true
|
||||||
|
}
|
||||||
|
if len(seen) < 100 {
|
||||||
|
t.Errorf("%d distinct prompts out of 100; the cache would answer the rest", len(seen))
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.SharedPrefix = true
|
||||||
|
first := prompt(cfg)
|
||||||
|
for range 20 {
|
||||||
|
if got := prompt(cfg); !strings.HasPrefix(got, first[:200]) {
|
||||||
|
t.Error("--shared-prefix did not produce a shared prefix")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptSizeFollowsTheRequest(t *testing.T) {
|
||||||
|
small := prompt(Config{PromptTokens: 32})
|
||||||
|
large := prompt(Config{PromptTokens: 512})
|
||||||
|
if len(large) < len(small)*4 {
|
||||||
|
t.Errorf("a 512-token prompt is %d chars against %d for 32", len(large), len(small))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigIsValidatedBeforeAnyLoadIsGenerated(t *testing.T) {
|
||||||
|
// The failure has to come before the traffic, not halfway through a ramp.
|
||||||
|
if _, err := Run(context.Background(), Config{Model: "m"}, nil); err == nil {
|
||||||
|
t.Error("a run with no target started")
|
||||||
|
}
|
||||||
|
if _, err := Run(context.Background(), Config{Target: "http://x"}, nil); err == nil {
|
||||||
|
t.Error("a run with no model started")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheRampStopsOnceTheSLOBreaks(t *testing.T) {
|
||||||
|
// Everything above a missed level is slower by construction, and finding
|
||||||
|
// that out costs real GPU time.
|
||||||
|
r := Result{}
|
||||||
|
r.Steps = []Step{
|
||||||
|
{Concurrency: 1, Completed: 1, MeetsSLO: true},
|
||||||
|
{Concurrency: 2, Completed: 1, MeetsSLO: false},
|
||||||
|
}
|
||||||
|
r.conclude()
|
||||||
|
if r.MaxConcurrencyAtSLO != 1 {
|
||||||
|
t.Errorf("capacity = %d", r.MaxConcurrencyAtSLO)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRoundTo(t *testing.T) {
|
||||||
|
cases := map[float64]float64{0.71: 0.70, 0.73: 0.75, 0.88: 0.90, 0.0: 0.0}
|
||||||
|
for in, want := range cases {
|
||||||
|
if got := roundTo(in, 0.05); got != want {
|
||||||
|
t.Errorf("roundTo(%v) = %v, want %v", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
package bench
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scrape"
|
||||||
|
)
|
||||||
|
|
||||||
|
// sampler reads an engine's metrics during a step.
|
||||||
|
//
|
||||||
|
// It is what turns a latency measurement into something admission can use: the
|
||||||
|
// queue depth and cache pressure observed while the SLO was still being met are
|
||||||
|
// the values target_waiting and kv_ceiling should have, rather than the round
|
||||||
|
// numbers somebody typed.
|
||||||
|
//
|
||||||
|
// Optional, because a run pointed at the gateway has no single engine to read.
|
||||||
|
type sampler struct {
|
||||||
|
url string
|
||||||
|
engine string
|
||||||
|
client *http.Client
|
||||||
|
cancel context.CancelFunc
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
wg sync.WaitGroup
|
||||||
|
waiting float64
|
||||||
|
running float64
|
||||||
|
kv float64
|
||||||
|
n int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newSampler(metricsURL, engine string) *sampler {
|
||||||
|
return &sampler{
|
||||||
|
url: metricsURL,
|
||||||
|
engine: engine,
|
||||||
|
client: &http.Client{Timeout: 2 * time.Second},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *sampler) start(ctx context.Context) {
|
||||||
|
if s.url == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx, s.cancel = context.WithCancel(ctx)
|
||||||
|
s.wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer s.wg.Done()
|
||||||
|
t := time.NewTicker(time.Second)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-t.C:
|
||||||
|
s.sample(ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *sampler) sample(ctx context.Context) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
resp, err := s.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// The same mapping the router reads with, so a bench result cannot disagree
|
||||||
|
// with what the scheduler sees. The engine has to be named: reading a
|
||||||
|
// llama.cpp endpoint with vLLM's names finds nothing, and the run would
|
||||||
|
// report a queue depth of zero and suggest thresholds built on it.
|
||||||
|
stats, err := scrape.Parse(resp.Body, scrape.MappingFor(s.engine))
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
s.waiting += float64(stats.Waiting)
|
||||||
|
s.running += float64(stats.Running)
|
||||||
|
s.kv += stats.KVCacheUsage
|
||||||
|
s.n++
|
||||||
|
s.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *sampler) stop() {
|
||||||
|
if s.cancel != nil {
|
||||||
|
s.cancel()
|
||||||
|
}
|
||||||
|
s.wg.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *sampler) averages() (waiting, running, kv float64) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.n == 0 {
|
||||||
|
return 0, 0, 0
|
||||||
|
}
|
||||||
|
n := float64(s.n)
|
||||||
|
return s.waiting / n, s.running / n, s.kv / n
|
||||||
|
}
|
||||||
@@ -0,0 +1,192 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"syscall"
|
||||||
|
"text/tabwriter"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newAgentCmd() *cobra.Command {
|
||||||
|
var configPath string
|
||||||
|
|
||||||
|
var (
|
||||||
|
server string
|
||||||
|
nodeID string
|
||||||
|
mgmtAddr string
|
||||||
|
)
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "agent",
|
||||||
|
Short: "Node agent: discovery and engine supervision",
|
||||||
|
Long: "Runs on every GPU node. It discovers the hardware, connects to the control\n" +
|
||||||
|
"plane, and starts, supervises and stops inference engines on demand.\n\n" +
|
||||||
|
"The agent dials out and keeps one stream open. Engines keep serving if the\n" +
|
||||||
|
"agent restarts or the control plane goes away: the router talks to them\n" +
|
||||||
|
"directly, and orphan adoption picks them up again.",
|
||||||
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
log := logger()
|
||||||
|
|
||||||
|
cfg, err := config.LoadAgent(configPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if server != "" {
|
||||||
|
cfg.Server = server
|
||||||
|
}
|
||||||
|
if nodeID != "" {
|
||||||
|
cfg.NodeID = nodeID
|
||||||
|
}
|
||||||
|
if mgmtAddr != "" {
|
||||||
|
cfg.MgmtAddr = mgmtAddr
|
||||||
|
}
|
||||||
|
if cfg.NodeID == "" {
|
||||||
|
host, err := os.Hostname()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("node_id is unset and the hostname is unavailable: %w", err)
|
||||||
|
}
|
||||||
|
cfg.NodeID = host
|
||||||
|
}
|
||||||
|
if cfg.Hostname == "" {
|
||||||
|
cfg.Hostname, _ = os.Hostname()
|
||||||
|
}
|
||||||
|
cfg.Version = version.Version
|
||||||
|
|
||||||
|
httpClient, err := agent.Client(cfg.TLS, 30*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
a, err := agent.New(cfg, httpClient, log)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
log.Info("agent starting",
|
||||||
|
"node", cfg.NodeID, "server", cfg.Server,
|
||||||
|
"fabrics", len(cfg.Fabrics), "nvml", discovery.NVMLSupported())
|
||||||
|
|
||||||
|
err = a.Run(ctx)
|
||||||
|
if errors.Is(err, context.Canceled) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cmd.Flags().StringVar(&configPath, "config", "", "configuration file")
|
||||||
|
cmd.Flags().StringVar(&server, "server", "", "control plane address (overrides the config)")
|
||||||
|
cmd.Flags().StringVar(&nodeID, "node-id", "", "node id (defaults to the hostname)")
|
||||||
|
cmd.Flags().StringVar(&mgmtAddr, "mgmt-addr", "", "address engines are reachable on")
|
||||||
|
cmd.AddCommand(newInventoryCmd())
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func newInventoryCmd() *cobra.Command {
|
||||||
|
var (
|
||||||
|
asJSON bool
|
||||||
|
smiBin string
|
||||||
|
ibRoot string
|
||||||
|
skipGPU bool
|
||||||
|
useNVML bool
|
||||||
|
)
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "inventory",
|
||||||
|
Short: "Show this node's GPUs and InfiniBand adapters",
|
||||||
|
Long: "Reports what the scheduler would see on this node: GPU classes, health,\n" +
|
||||||
|
"and the fabric each adapter is attached to.\n\n" +
|
||||||
|
"Run it before adding a node to the cluster — it is the fastest way to find\n" +
|
||||||
|
"a card another process is already using, or a port with no subnet manager.",
|
||||||
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
inv, err := discovery.Discover(cmd.Context(), discovery.Options{
|
||||||
|
NvidiaSMI: smiBin,
|
||||||
|
IBRoot: ibRoot,
|
||||||
|
PreferNVML: useNVML,
|
||||||
|
SkipGPUs: skipGPU,
|
||||||
|
}, logger())
|
||||||
|
if err != nil {
|
||||||
|
// Not fatal: a node whose GPUs cannot be read still has IB
|
||||||
|
// worth showing, and the error is itself the useful output.
|
||||||
|
fmt.Fprintf(os.Stderr, "%v\n", err)
|
||||||
|
inv.Source = "unavailable"
|
||||||
|
}
|
||||||
|
|
||||||
|
if asJSON {
|
||||||
|
enc := json.NewEncoder(os.Stdout)
|
||||||
|
enc.SetIndent("", " ")
|
||||||
|
return enc.Encode(inv)
|
||||||
|
}
|
||||||
|
printInventory(inv)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
f := cmd.Flags()
|
||||||
|
f.BoolVar(&asJSON, "json", false, "emit JSON")
|
||||||
|
f.StringVar(&smiBin, "nvidia-smi", "", "path to nvidia-smi")
|
||||||
|
f.StringVar(&ibRoot, "ib-root", discovery.SysfsRoot, "sysfs InfiniBand directory")
|
||||||
|
f.BoolVar(&skipGPU, "no-gpu", false, "skip GPU discovery")
|
||||||
|
f.BoolVar(&useNVML, "nvml", true, "prefer the NVML backend when this build has one (it is the only source of NVLink topology)")
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func printInventory(inv discovery.Inventory) {
|
||||||
|
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||||
|
|
||||||
|
fmt.Fprintf(w, "GPUs (%d, via %s)\n", len(inv.GPUs), inv.Source)
|
||||||
|
if len(inv.GPUs) == 0 {
|
||||||
|
fmt.Fprintln(w, " none")
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(w, " IDX\tCLASS\tNAME\tMEM\tCC\tHEALTH")
|
||||||
|
for _, g := range inv.GPUs {
|
||||||
|
health := "ok"
|
||||||
|
if !g.Health.OK() {
|
||||||
|
health = g.Health.Problems[0]
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, " %d\t%s\t%s\t%d MiB\t%s\t%s\n",
|
||||||
|
g.Index, g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, " classes: %v\n", inv.Classes())
|
||||||
|
if !discovery.NVMLSupported() {
|
||||||
|
fmt.Fprintln(w, " NVLink topology unavailable: rebuild with -tags nvml.")
|
||||||
|
fmt.Fprintln(w, " Without it, placement cannot keep a tensor-parallel group inside")
|
||||||
|
fmt.Fprintln(w, " one NVLink domain (plan §6.2).")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintf(w, "\nInfiniBand adapters (%d)\n", len(inv.HCAs))
|
||||||
|
if len(inv.HCAs) == 0 {
|
||||||
|
// Worth stating plainly: such a node is its own single-node island and
|
||||||
|
// can never host a multi-node replica.
|
||||||
|
fmt.Fprintln(w, " none — this node cannot host multi-node replicas")
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(w, " HCA\tPORT\tSTATE\tRATE\tLINK\tGID PREFIX\tNUMA")
|
||||||
|
for _, h := range inv.HCAs {
|
||||||
|
for _, p := range h.Ports {
|
||||||
|
fmt.Fprintf(w, " %s\t%d\t%s\t%d Gb/s\t%s\t%s\t%d\n",
|
||||||
|
h.Name, p.Num, p.State, p.RateGbps, p.LinkLayer, p.GIDPrefix, h.NUMANode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Fprintln(w, "\n Fabric identity is not derived from the GID prefix by default:")
|
||||||
|
fmt.Fprintln(w, " OpenSM's default is fe80:: on every fabric, so a shared prefix proves")
|
||||||
|
fmt.Fprintln(w, " nothing. Label fabrics explicitly in agent.yaml (plan §5.2).")
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = w.Flush()
|
||||||
|
}
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newBenchCmd(asJSON *bool) *cobra.Command {
|
||||||
|
cfg := bench.DefaultConfig()
|
||||||
|
var (
|
||||||
|
steps []int
|
||||||
|
target string
|
||||||
|
metrics string
|
||||||
|
engine string
|
||||||
|
key string
|
||||||
|
)
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "bench <model>",
|
||||||
|
Short: "Measure what a replica can do, and what the router should be told",
|
||||||
|
Long: "Drives real traffic. Point it at a replica to measure that hardware, or at\n" +
|
||||||
|
"the gateway to measure the whole path — they answer different questions,\n" +
|
||||||
|
"and only the first produces a capacity profile.\n\n" +
|
||||||
|
"Concurrency climbs until the SLO breaks. The last level that still met it\n" +
|
||||||
|
"is the capacity: peak throughput arrives long after latency has become\n" +
|
||||||
|
"unusable, so throughput alone would overstate what the hardware can serve.\n\n" +
|
||||||
|
"This costs real GPU time and competes with real traffic. Run it on a\n" +
|
||||||
|
"replica you have cordoned, or during a window where that is acceptable.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
cfg.Model = args[0]
|
||||||
|
cfg.Target = target
|
||||||
|
cfg.MetricsURL = metrics
|
||||||
|
cfg.Engine = engine
|
||||||
|
cfg.APIKey = key
|
||||||
|
if len(steps) > 0 {
|
||||||
|
cfg.Concurrency = steps
|
||||||
|
}
|
||||||
|
|
||||||
|
if metrics == "" {
|
||||||
|
cmd.PrintErrln("No --metrics endpoint: queue depth and KV usage will not be " +
|
||||||
|
"measured, so target_waiting and kv_ceiling cannot be suggested.")
|
||||||
|
}
|
||||||
|
cmd.PrintErrf("benchmarking %s against %s\n\n", cfg.Model, cfg.Target)
|
||||||
|
|
||||||
|
// Fixed columns rather than a tabwriter: rows are printed as each
|
||||||
|
// level finishes, and a tabwriter cannot align a column it has not
|
||||||
|
// seen the rest of yet — it lays out whatever has been flushed, so
|
||||||
|
// every row after the first came out ragged.
|
||||||
|
const row = "%-5s %-7s %-7s %-9s %-9s %-9s %-6s %-5s %s\n"
|
||||||
|
fmt.Fprintf(cmd.ErrOrStderr(), row,
|
||||||
|
"CONC", "REQ/S", "TOK/S", "TTFT p50", "TTFT p99", "TPOT p99", "WAIT", "KV", "SLO")
|
||||||
|
|
||||||
|
progress := func(s bench.Step) {
|
||||||
|
verdict := "ok"
|
||||||
|
if !s.MeetsSLO {
|
||||||
|
verdict = "MISSED"
|
||||||
|
}
|
||||||
|
if s.Failed > 0 {
|
||||||
|
verdict = fmt.Sprintf("%d failed", s.Failed)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.ErrOrStderr(), row,
|
||||||
|
fmt.Sprint(s.Concurrency),
|
||||||
|
fmt.Sprintf("%.1f", s.RequestsPerSec),
|
||||||
|
fmt.Sprintf("%.0f", s.TokensPerSec),
|
||||||
|
ms(s.TTFT.P50), ms(s.TTFT.P99), ms(s.TPOT.P99),
|
||||||
|
fmt.Sprintf("%.1f", s.Waiting),
|
||||||
|
fmt.Sprintf("%.0f%%", s.KVUsage*100),
|
||||||
|
verdict)
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := bench.Run(cmd.Context(), cfg, progress)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
return json.NewEncoder(cmd.OutOrStdout()).Encode(res)
|
||||||
|
}
|
||||||
|
printConclusion(cmd, res, cfg)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
f := cmd.Flags()
|
||||||
|
f.StringVar(&target, "target", "http://127.0.0.1:8080", "base URL: a replica, or the gateway")
|
||||||
|
f.StringVar(&metrics, "metrics", "", "the replica's /metrics, to observe queue depth and KV usage")
|
||||||
|
f.StringVar(&engine, "engine", "", "which metric names --metrics uses: vllm | sglang | llamacpp")
|
||||||
|
f.StringVar(&key, "key", "", "API key, when the gateway requires one")
|
||||||
|
f.IntSliceVar(&steps, "concurrency", nil, "levels to try (default 1,2,4,…,128)")
|
||||||
|
f.DurationVar(&cfg.Duration, "duration", cfg.Duration, "how long to hold each level")
|
||||||
|
f.IntVar(&cfg.PromptTokens, "prompt-tokens", cfg.PromptTokens, "approximate prompt size")
|
||||||
|
f.IntVar(&cfg.MaxTokens, "max-tokens", cfg.MaxTokens, "tokens to generate per request")
|
||||||
|
f.BoolVar(&cfg.SharedPrefix, "shared-prefix", false,
|
||||||
|
"send every request the same prompt, measuring the cached case instead of the hardware")
|
||||||
|
f.DurationVar(&cfg.SLO.TTFTp99, "ttft-p99", cfg.SLO.TTFTp99, "time-to-first-token target")
|
||||||
|
f.DurationVar(&cfg.SLO.TPOTp99, "tpot-p99", cfg.SLO.TPOTp99, "per-output-token target")
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func printConclusion(cmd *cobra.Command, res bench.Result, cfg bench.Config) {
|
||||||
|
out := cmd.OutOrStdout()
|
||||||
|
fmt.Fprintln(out)
|
||||||
|
|
||||||
|
if res.MaxConcurrencyAtSLO == 0 {
|
||||||
|
fmt.Fprintf(out, "No level met the SLO (TTFT p99 ≤ %s, TPOT p99 ≤ %s).\n",
|
||||||
|
cfg.SLO.TTFTp99, cfg.SLO.TPOTp99)
|
||||||
|
if len(res.Steps) > 0 && res.Steps[0].FirstError != "" {
|
||||||
|
fmt.Fprintf(out, "First error: %s\n", res.Steps[0].FirstError)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "Either the hardware cannot serve this model at this SLO, or the")
|
||||||
|
fmt.Fprintln(out, "target is stricter than the model can meet at any concurrency.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintf(out, "Capacity: %d concurrent requests at %.0f tokens/s\n",
|
||||||
|
res.MaxConcurrencyAtSLO, res.ThroughputTokPerSec)
|
||||||
|
fmt.Fprintf(out, " TTFT p99 %s · TPOT p99 %s · measured over %s\n\n",
|
||||||
|
ms(res.TTFTp99), ms(res.TPOTp99), res.Elapsed.Round(time.Second))
|
||||||
|
|
||||||
|
fmt.Fprintln(out, "Put this in the model's profile:")
|
||||||
|
fmt.Fprintf(out, " weight: %d\n", res.MaxConcurrencyAtSLO)
|
||||||
|
if res.SuggestedTargetWaiting > 0 || res.SuggestedKVCeiling > 0 {
|
||||||
|
fmt.Fprintln(out, "\nAnd in router.scoring, from what the engine reported while still")
|
||||||
|
fmt.Fprintln(out, "meeting the target:")
|
||||||
|
if res.SuggestedTargetWaiting > 0 {
|
||||||
|
fmt.Fprintf(out, " target_waiting: %d\n", res.SuggestedTargetWaiting)
|
||||||
|
}
|
||||||
|
if res.SuggestedKVCeiling > 0 {
|
||||||
|
fmt.Fprintf(out, " kv_ceiling: %.2f\n", res.SuggestedKVCeiling)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if last := res.Steps[len(res.Steps)-1]; !last.MeetsSLO {
|
||||||
|
fmt.Fprintf(out, "\n%d concurrent was the first level to miss it (TTFT p99 %s).\n",
|
||||||
|
last.Concurrency, ms(last.TTFT.P99))
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(out, "\nEvery level tried met the SLO, so %d is a floor rather than a\n",
|
||||||
|
res.MaxConcurrencyAtSLO)
|
||||||
|
fmt.Fprintln(out, "ceiling — raise --concurrency to find where it actually breaks.")
|
||||||
|
}
|
||||||
|
if cfg.SharedPrefix {
|
||||||
|
fmt.Fprintln(out, "\nRun with --shared-prefix: this measures the engine's prefix cache,")
|
||||||
|
fmt.Fprintln(out, "not its hardware. Do not use these numbers as capacity.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func ms(d time.Duration) string {
|
||||||
|
switch {
|
||||||
|
case d == 0:
|
||||||
|
return "-"
|
||||||
|
case d < time.Millisecond:
|
||||||
|
return d.Round(10 * time.Microsecond).String()
|
||||||
|
case d < time.Second:
|
||||||
|
return d.Round(time.Millisecond).String()
|
||||||
|
default:
|
||||||
|
return d.Round(10 * time.Millisecond).String()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,254 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/x509"
|
||||||
|
"encoding/pem"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||||
|
)
|
||||||
|
|
||||||
|
// defaultPKIDir is where the authority lives on the machine that issues
|
||||||
|
// certificates. Only that machine ever holds ca-key.pem; everything else gets
|
||||||
|
// ca.pem and its own leaf.
|
||||||
|
const defaultPKIDir = "/etc/llmgw/pki"
|
||||||
|
|
||||||
|
func newCertCmd() *cobra.Command {
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "cert",
|
||||||
|
Short: "Issue the certificates the control plane and agents authenticate with",
|
||||||
|
Long: "llmgw's internal connections use mutual TLS, and a public authority cannot\n" +
|
||||||
|
"supply them: Let's Encrypt and its peers stopped issuing certificates\n" +
|
||||||
|
"carrying the clientAuth usage, so nothing they hand out can authenticate an\n" +
|
||||||
|
"agent. These commands run a small private authority instead.\n\n" +
|
||||||
|
"They work on files and talk to nothing, so they run wherever you keep the\n" +
|
||||||
|
"authority — not necessarily on a cluster machine.",
|
||||||
|
}
|
||||||
|
cmd.AddCommand(newInitCACmd(), newIssueServerCmd(), newIssueClientCmd(), newCertInspectCmd())
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func newInitCACmd() *cobra.Command {
|
||||||
|
var (
|
||||||
|
dir string
|
||||||
|
cn string
|
||||||
|
force bool
|
||||||
|
)
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "init-ca",
|
||||||
|
Short: "Create the cluster's certificate authority",
|
||||||
|
Long: "Writes ca.pem and ca-key.pem. Copy ca.pem to every machine; leave\n" +
|
||||||
|
"ca-key.pem here, because whoever holds it can mint any node's identity.",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
keyPath := filepath.Join(dir, "ca-key.pem")
|
||||||
|
if _, err := os.Stat(keyPath); err == nil && !force {
|
||||||
|
// Overwriting the key orphans every certificate ever issued from
|
||||||
|
// it, across every machine, with no warning until connections
|
||||||
|
// start failing.
|
||||||
|
return fmt.Errorf("%s already exists; re-running would invalidate every "+
|
||||||
|
"certificate issued from it (pass --force if that is what you want)", keyPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
ca, err := tlsconf.NewCA(cn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := ca.Write(dir); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
cmd.Printf("authority %q written to %s\n", cn, dir)
|
||||||
|
cmd.Printf(" ca.pem trust this everywhere (valid until %s)\n",
|
||||||
|
ca.Cert.NotAfter.Format(time.DateOnly))
|
||||||
|
cmd.Printf(" ca-key.pem keep it here, mode 0600\n")
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cmd.Flags().StringVar(&dir, "dir", defaultPKIDir, "directory holding the authority")
|
||||||
|
cmd.Flags().StringVar(&cn, "cn", "llmgw cluster CA", "common name for the authority")
|
||||||
|
cmd.Flags().BoolVar(&force, "force", false, "replace an existing authority")
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func newIssueServerCmd() *cobra.Command {
|
||||||
|
var (
|
||||||
|
dir string
|
||||||
|
out string
|
||||||
|
hosts []string
|
||||||
|
name string
|
||||||
|
)
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "issue-server <common-name>",
|
||||||
|
Short: "Issue the control plane's certificate",
|
||||||
|
Long: "Every address an agent might dial must appear in --host, including bare\n" +
|
||||||
|
"IP addresses. Hostname verification reads subject alternative names and\n" +
|
||||||
|
"ignores the common name entirely, so a certificate that names the host\n" +
|
||||||
|
"only in its subject is rejected by every client.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
ca, err := loadCA(dir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
leaf, err := ca.IssueServer(args[0], hosts)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if name == "" {
|
||||||
|
name = "server"
|
||||||
|
}
|
||||||
|
if err := leaf.Write(out, name); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return describeIssued(cmd, out, name, hosts)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cmd.Flags().StringVar(&dir, "dir", defaultPKIDir, "directory holding the authority")
|
||||||
|
cmd.Flags().StringVar(&out, "out", ".", "directory to write the certificate into")
|
||||||
|
cmd.Flags().StringVar(&name, "name", "server", "base filename for the certificate")
|
||||||
|
cmd.Flags().StringArrayVar(&hosts, "host", nil, "address agents dial; repeatable (IP or DNS name)")
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func newIssueClientCmd() *cobra.Command {
|
||||||
|
var dir, out string
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "issue-client <node-id>",
|
||||||
|
Short: "Issue one agent's certificate",
|
||||||
|
Long: "The node id becomes the certificate's common name, and the control plane\n" +
|
||||||
|
"refuses any agent whose claimed id does not match it. That is what makes a\n" +
|
||||||
|
"node's identity something it proves rather than something it picks.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
ca, err := loadCA(dir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
leaf, err := ca.IssueClient(args[0])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := leaf.Write(out, args[0]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return describeIssued(cmd, out, args[0], nil)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cmd.Flags().StringVar(&dir, "dir", defaultPKIDir, "directory holding the authority")
|
||||||
|
cmd.Flags().StringVar(&out, "out", ".", "directory to write the certificate into")
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func newCertInspectCmd() *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "inspect <file.pem>",
|
||||||
|
Short: "Print what a certificate actually says",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
cert, err := readCert(args[0])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.Printf("subject %s\n", cert.Subject)
|
||||||
|
cmd.Printf("issuer %s\n", cert.Issuer)
|
||||||
|
cmd.Printf("valid %s .. %s", cert.NotBefore.Format(time.DateOnly),
|
||||||
|
cert.NotAfter.Format(time.DateOnly))
|
||||||
|
if d := time.Until(cert.NotAfter); d < 0 {
|
||||||
|
cmd.Printf(" (EXPIRED)")
|
||||||
|
} else {
|
||||||
|
cmd.Printf(" (%d days left)", int(d.Hours()/24))
|
||||||
|
}
|
||||||
|
cmd.Println()
|
||||||
|
|
||||||
|
if cert.IsCA {
|
||||||
|
cmd.Println("role certificate authority")
|
||||||
|
}
|
||||||
|
if u := usageNames(cert.ExtKeyUsage); u != "" {
|
||||||
|
cmd.Printf("usage %s\n", u)
|
||||||
|
}
|
||||||
|
if sans := sanNames(cert); sans != "" {
|
||||||
|
cmd.Printf("hosts %s\n", sans)
|
||||||
|
} else if !cert.IsCA && !hasUsage(cert, x509.ExtKeyUsageClientAuth) {
|
||||||
|
cmd.Println("hosts none — no client will accept this as a server certificate")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadCA(dir string) (*tlsconf.CA, error) {
|
||||||
|
ca, err := tlsconf.LoadCA(filepath.Join(dir, "ca.pem"), filepath.Join(dir, "ca-key.pem"))
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil, fmt.Errorf("no authority in %s — run 'llmgw ctl cert init-ca --dir %s' first", dir, dir)
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return ca, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func describeIssued(cmd *cobra.Command, dir, name string, hosts []string) error {
|
||||||
|
cert, err := readCert(filepath.Join(dir, name+".pem"))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
cmd.Printf("issued %s\n", cert.Subject.CommonName)
|
||||||
|
cmd.Printf(" %s\n", filepath.Join(dir, name+".pem"))
|
||||||
|
cmd.Printf(" %s\n", filepath.Join(dir, name+"-key.pem"))
|
||||||
|
if len(hosts) > 0 {
|
||||||
|
cmd.Printf(" hosts: %s\n", strings.Join(hosts, ", "))
|
||||||
|
}
|
||||||
|
cmd.Printf(" expires %s\n", cert.NotAfter.Format(time.DateOnly))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readCert(path string) (*x509.Certificate, error) {
|
||||||
|
b, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
block, _ := pem.Decode(b)
|
||||||
|
if block == nil {
|
||||||
|
return nil, fmt.Errorf("%s is not PEM", path)
|
||||||
|
}
|
||||||
|
return x509.ParseCertificate(block.Bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasUsage(cert *x509.Certificate, want x509.ExtKeyUsage) bool {
|
||||||
|
for _, u := range cert.ExtKeyUsage {
|
||||||
|
if u == want {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func usageNames(usages []x509.ExtKeyUsage) string {
|
||||||
|
var out []string
|
||||||
|
for _, u := range usages {
|
||||||
|
switch u {
|
||||||
|
case x509.ExtKeyUsageServerAuth:
|
||||||
|
out = append(out, "serverAuth (identifies the control plane)")
|
||||||
|
case x509.ExtKeyUsageClientAuth:
|
||||||
|
out = append(out, "clientAuth (identifies an agent)")
|
||||||
|
default:
|
||||||
|
out = append(out, fmt.Sprintf("%v", u))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(out, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func sanNames(cert *x509.Certificate) string {
|
||||||
|
names := append([]string{}, cert.DNSNames...)
|
||||||
|
for _, ip := range cert.IPAddresses {
|
||||||
|
names = append(names, ip.String())
|
||||||
|
}
|
||||||
|
return strings.Join(names, ", ")
|
||||||
|
}
|
||||||
@@ -0,0 +1,427 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"text/tabwriter"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ctlClient talks to a server's admin API.
|
||||||
|
type ctlClient struct {
|
||||||
|
base string
|
||||||
|
http *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
func newCtlClient(addr string) *ctlClient {
|
||||||
|
if !strings.HasPrefix(addr, "http://") && !strings.HasPrefix(addr, "https://") {
|
||||||
|
addr = "http://" + addr
|
||||||
|
}
|
||||||
|
return &ctlClient{
|
||||||
|
base: strings.TrimRight(addr, "/"),
|
||||||
|
http: &http.Client{Timeout: 15 * time.Second},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *ctlClient) get(ctx context.Context, path string, out any) error {
|
||||||
|
return c.do(ctx, http.MethodGet, path, nil, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// post sends an already-encoded body, so a caller that has marshalled a typed
|
||||||
|
// value keeps its field names rather than having them re-derived here.
|
||||||
|
func (c *ctlClient) post(ctx context.Context, path string, body []byte, out any) error {
|
||||||
|
return c.do(ctx, http.MethodPost, path, json.RawMessage(body), out)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *ctlClient) delete(ctx context.Context, path string, out any) error {
|
||||||
|
return c.do(ctx, http.MethodDelete, path, nil, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *ctlClient) do(ctx context.Context, method, path string, body, out any) error {
|
||||||
|
var rdr io.Reader
|
||||||
|
if body != nil {
|
||||||
|
b, err := json.Marshal(body)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rdr = bytes.NewReader(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, method, c.base+path, rdr)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if body != nil {
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
}
|
||||||
|
// Who made the change, kept with a stored model so a catalog that grew
|
||||||
|
// over months can say where each entry came from.
|
||||||
|
req.Header.Set("X-Llmgw-Actor", actor())
|
||||||
|
|
||||||
|
resp, err := c.http.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reaching the admin API at %s: %w\n"+
|
||||||
|
"(is the server running, and is --server pointing at its admin_listen port?)", c.base, err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
var e struct {
|
||||||
|
Error string `json:"error"`
|
||||||
|
}
|
||||||
|
b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<12))
|
||||||
|
if json.Unmarshal(b, &e) == nil && e.Error != "" {
|
||||||
|
return fmt.Errorf("%s: %s", resp.Status, e.Error)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(b)))
|
||||||
|
}
|
||||||
|
if out == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return json.NewDecoder(resp.Body).Decode(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newCtlCmd() *cobra.Command {
|
||||||
|
var (
|
||||||
|
server string
|
||||||
|
asJSON bool
|
||||||
|
)
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "ctl",
|
||||||
|
Short: "Inspect and operate a running cluster",
|
||||||
|
Long: "Talks to a server's admin API, which listens on admin_listen (:9090 by\n" +
|
||||||
|
"default) rather than the client-facing port.",
|
||||||
|
}
|
||||||
|
cmd.PersistentFlags().StringVar(&server, "server", "127.0.0.1:9090", "admin API address")
|
||||||
|
cmd.PersistentFlags().BoolVar(&asJSON, "json", false, "emit JSON")
|
||||||
|
|
||||||
|
client := func() *ctlClient { return newCtlClient(server) }
|
||||||
|
|
||||||
|
cmd.AddCommand(
|
||||||
|
newNodesCmd(client, &asJSON),
|
||||||
|
newModelsCmd(client, &asJSON),
|
||||||
|
newReplicasCmd(client, &asJSON),
|
||||||
|
newExplainCmd(client, &asJSON),
|
||||||
|
newNodeStateCmd(client, "cordon", "cordoned",
|
||||||
|
"Stop placing new work on a node; what runs there keeps running"),
|
||||||
|
newNodeStateCmd(client, "uncordon", "ready",
|
||||||
|
"Allow placement on a node again"),
|
||||||
|
newNodeStateCmd(client, "drain", "draining",
|
||||||
|
"Move work off a node, one replica at a time"),
|
||||||
|
// cert works on files rather than the admin API, so --server does not
|
||||||
|
// apply to it; it lives here because this is where operators look.
|
||||||
|
newBenchCmd(&asJSON),
|
||||||
|
newStatusCmd(client, &asJSON),
|
||||||
|
newModelCmd(client),
|
||||||
|
newEngineCheckCmd(),
|
||||||
|
newToolCheckCmd(),
|
||||||
|
newWeightsCmd(client, &asJSON),
|
||||||
|
newCertCmd(),
|
||||||
|
)
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "nodes",
|
||||||
|
Short: "List nodes and their capacity",
|
||||||
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
var nodes []adminapi.NodeView
|
||||||
|
if err := client().get(cmd.Context(), "/api/v1/nodes", &nodes); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
return emitJSON(nodes)
|
||||||
|
}
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
fmt.Println("No nodes have registered.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
w := tab()
|
||||||
|
fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tDISK\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
|
||||||
|
for _, n := range nodes {
|
||||||
|
fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%d\t%s\n",
|
||||||
|
n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB),
|
||||||
|
diskColumn(n.DiskFreeGB, n.DiskTotalGB),
|
||||||
|
join(n.Classes), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
|
||||||
|
}
|
||||||
|
_ = w.Flush()
|
||||||
|
|
||||||
|
// Unhealthy GPUs are usually the answer when a node looks idle but
|
||||||
|
// nothing schedules onto it.
|
||||||
|
for _, n := range nodes {
|
||||||
|
for _, p := range n.Unhealthy {
|
||||||
|
fmt.Printf("\n%s: %s (kept out of placement)", n.ID, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if hasUnhealthy(nodes) {
|
||||||
|
fmt.Println()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// modelStatus says what is happening to a model, in the words that decide what
|
||||||
|
// to do about it.
|
||||||
|
//
|
||||||
|
// The previous rendering distinguished "a replica is coming up" from "nothing
|
||||||
|
// could be placed" by a parenthetical — "0/1 ready (1 total)" against "0/1
|
||||||
|
// ready" — and those need opposite responses: one means wait, the other means
|
||||||
|
// look at explain-placement. On four models the difference is findable; on
|
||||||
|
// forty it is invisible, and a model that will never start looks exactly like
|
||||||
|
// one that is thirty seconds away.
|
||||||
|
//
|
||||||
|
// A model appearing here before it serves is deliberate: this is the operator's
|
||||||
|
// view, and a model coming up is precisely what someone wants to see. Clients
|
||||||
|
// are told separately and only when it is true — /v1/models lists what is
|
||||||
|
// actually served, never what is merely catalogued.
|
||||||
|
func modelStatus(m adminapi.ModelView) string {
|
||||||
|
switch {
|
||||||
|
case m.Min == 0 && m.Total == 0:
|
||||||
|
return "stopped"
|
||||||
|
case m.Ready >= m.Min && m.Min > 0:
|
||||||
|
return "ready"
|
||||||
|
case m.Total > m.Ready:
|
||||||
|
return "starting"
|
||||||
|
default:
|
||||||
|
return "not placed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// idleColumn shows how long a replica has gone unused.
|
||||||
|
//
|
||||||
|
// "-" for a replica nothing has reported use of, which is emphatically not the
|
||||||
|
// same as idle forever: it is what every replica looks like before its first
|
||||||
|
// request, and what they all look like after a router restart. Showing that as
|
||||||
|
// a very large number would make an untouched fleet read as an abandoned one.
|
||||||
|
func idleColumn(sec int64) string {
|
||||||
|
if sec < 0 {
|
||||||
|
return "-"
|
||||||
|
}
|
||||||
|
return (time.Duration(sec) * time.Second).Round(time.Second).String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// memColumn shows a node's memory, distinguishing "did not say" from a number.
|
||||||
|
// An older agent reports nothing, and that is not the same as a small machine.
|
||||||
|
func memColumn(gb int) string {
|
||||||
|
if gb <= 0 {
|
||||||
|
return "-"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%dG", gb)
|
||||||
|
}
|
||||||
|
|
||||||
|
// diskColumn renders free space against the size it is free out of.
|
||||||
|
//
|
||||||
|
// Both, because they answer different questions and the first is useless
|
||||||
|
// without the second: 8 GB free is comfortable on a 10 TB filer and nearly
|
||||||
|
// full on a 10 GB root partition, and it is the second case that stops a
|
||||||
|
// model from being placed.
|
||||||
|
//
|
||||||
|
// A dash for a node that reported nothing, which an older agent and a node
|
||||||
|
// with no model directories both do. It has to read as "not known" rather
|
||||||
|
// than as "no space", or every unconfigured node looks like it is full.
|
||||||
|
func diskColumn(freeGB, totalGB int) string {
|
||||||
|
if totalGB <= 0 {
|
||||||
|
return "-"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%dG/%dG", freeGB, totalGB)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "models",
|
||||||
|
Short: "List the catalog and how many replicas each model has",
|
||||||
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
var models []adminapi.ModelView
|
||||||
|
if err := client().get(cmd.Context(), "/api/v1/models", &models); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
return emitJSON(models)
|
||||||
|
}
|
||||||
|
if len(models) == 0 {
|
||||||
|
fmt.Println("The catalog is empty.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
w := tab()
|
||||||
|
fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tALIASES\tPROFILES\tFALLBACKS")
|
||||||
|
for _, m := range models {
|
||||||
|
serving := "no"
|
||||||
|
if m.Ready > 0 {
|
||||||
|
serving = "yes"
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\n",
|
||||||
|
m.Name, serving, m.Ready, m.Min, modelStatus(m),
|
||||||
|
join(m.Aliases), join(m.Profiles), join(m.Fallbacks))
|
||||||
|
}
|
||||||
|
return w.Flush()
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||||
|
var model string
|
||||||
|
c := &cobra.Command{
|
||||||
|
Use: "replicas",
|
||||||
|
Short: "List replicas and the shards they are made of",
|
||||||
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
path := "/api/v1/replicas"
|
||||||
|
if model != "" {
|
||||||
|
path += "?model=" + model
|
||||||
|
}
|
||||||
|
var reps []adminapi.ReplicaView
|
||||||
|
if err := client().get(cmd.Context(), path, &reps); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
return emitJSON(reps)
|
||||||
|
}
|
||||||
|
if len(reps) == 0 {
|
||||||
|
fmt.Println("No replicas are running.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
w := tab()
|
||||||
|
fmt.Fprintln(w, "REPLICA\tMODEL\tSTATE\tPOOL\tNODES\tAGE\tIDLE\tENDPOINT")
|
||||||
|
for _, r := range reps {
|
||||||
|
var nodes []string
|
||||||
|
for _, s := range r.Shards {
|
||||||
|
nodes = append(nodes, fmt.Sprintf("%s:%v", s.Node, s.GPUs))
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||||
|
r.ID, r.Model, r.State, r.Pool, join(nodes),
|
||||||
|
(time.Duration(r.AgeSec) * time.Second).String(),
|
||||||
|
idleColumn(r.IdleSec), r.Endpoint)
|
||||||
|
}
|
||||||
|
_ = w.Flush()
|
||||||
|
|
||||||
|
for _, r := range reps {
|
||||||
|
if r.Err != "" {
|
||||||
|
fmt.Printf("\n%s: %s\n", r.ID, r.Err)
|
||||||
|
}
|
||||||
|
for _, s := range r.Shards {
|
||||||
|
if s.Err != "" {
|
||||||
|
fmt.Printf(" %s on %s: %s\n", s.ID, s.Node, s.Err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c.Flags().StringVar(&model, "model", "", "only this model's replicas")
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func newExplainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "explain-placement <model>",
|
||||||
|
Short: "Explain why a model has not been placed",
|
||||||
|
Long: "Prints every candidate placement considered and the reason it was\n" +
|
||||||
|
"rejected. \"Unschedulable\" on its own says nothing about which of six\n" +
|
||||||
|
"constraints bit.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
var v adminapi.ExplainView
|
||||||
|
if err := client().get(cmd.Context(),
|
||||||
|
"/api/v1/explain-placement?model="+args[0], &v); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
return emitJSON(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
if v.Placed && len(v.Rejections) == 0 {
|
||||||
|
fmt.Printf("%s is placed; nothing was rejected.\n", v.Model)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if len(v.Rejections) == 0 {
|
||||||
|
fmt.Printf("No placement has been attempted for %s yet.\n", v.Model)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if v.Placed {
|
||||||
|
fmt.Printf("%s is placed. The candidates rejected along the way:\n\n", v.Model)
|
||||||
|
} else {
|
||||||
|
fmt.Printf("%s could not be placed. Every candidate and why:\n\n", v.Model)
|
||||||
|
}
|
||||||
|
w := tab()
|
||||||
|
fmt.Fprintln(w, "PROFILE\tPOOL\tNODE\tREASON")
|
||||||
|
for _, r := range v.Rejections {
|
||||||
|
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", r.Profile, dash(r.Pool), dash(r.Node), r.Reason)
|
||||||
|
}
|
||||||
|
return w.Flush()
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newNodeStateCmd(client func() *ctlClient, use, state, short string) *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: use + " <node>",
|
||||||
|
Short: short,
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
var out map[string]string
|
||||||
|
err := client().do(cmd.Context(), http.MethodPost,
|
||||||
|
"/api/v1/nodes/"+args[0]+"/state",
|
||||||
|
map[string]string{"state": state}, &out)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s is now %s\n", out["id"], out["state"])
|
||||||
|
if state == "draining" {
|
||||||
|
fmt.Println("Replicas will be moved off one at a time, each replacement " +
|
||||||
|
"brought up before the original is stopped.")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- output helpers ---
|
||||||
|
|
||||||
|
func tab() *tabwriter.Writer {
|
||||||
|
return tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func emitJSON(v any) error {
|
||||||
|
enc := json.NewEncoder(os.Stdout)
|
||||||
|
enc.SetIndent("", " ")
|
||||||
|
return enc.Encode(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
func join(v []string) string {
|
||||||
|
if len(v) == 0 {
|
||||||
|
return "-"
|
||||||
|
}
|
||||||
|
return strings.Join(v, ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
func dash(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "-"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasUnhealthy(nodes []adminapi.NodeView) bool {
|
||||||
|
for _, n := range nodes {
|
||||||
|
if len(n.Unhealthy) > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||||
|
)
|
||||||
|
|
||||||
|
// newWeightsCmd reports and reclaims model files nothing needs.
|
||||||
|
//
|
||||||
|
// The dangerous half of storage management, so its shape is defensive: listing
|
||||||
|
// is the default and deleting takes a flag, only files llmgw fetched are ever
|
||||||
|
// candidates, and the node checks every path again before unlinking it.
|
||||||
|
//
|
||||||
|
// What it will never touch is the more important half of the contract. Weights
|
||||||
|
// an operator put on a node — by scp, by a shared mount, by hand — are none of
|
||||||
|
// this system's business however unreferenced they look, and a model that took
|
||||||
|
// a week to copy must not be removable by a command someone ran to tidy up.
|
||||||
|
func newWeightsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "weights",
|
||||||
|
Short: "Show cached model files no model in the catalog needs",
|
||||||
|
Long: "Lists weights llmgw downloaded for models that are no longer in the\n" +
|
||||||
|
"catalog, per node, with what deleting them would free.\n\n" +
|
||||||
|
"Only files llmgw fetched are considered. Anything an operator put on a\n" +
|
||||||
|
"node is left alone, whatever the catalog says — llmgw observes storage\n" +
|
||||||
|
"rather than owning it, and only takes back what it put there itself.\n\n" +
|
||||||
|
"Nothing is deleted without --collect.",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
}
|
||||||
|
var collect bool
|
||||||
|
var node string
|
||||||
|
|
||||||
|
cmd.Flags().BoolVar(&collect, "collect", false, "delete them, rather than only listing")
|
||||||
|
cmd.Flags().StringVar(&node, "node", "", "limit to one node")
|
||||||
|
|
||||||
|
cmd.RunE = func(cmd *cobra.Command, _ []string) error {
|
||||||
|
if collect {
|
||||||
|
return collectOrphans(cmd, client(), node, *asJSON)
|
||||||
|
}
|
||||||
|
return listOrphanWeights(cmd, client(), node, *asJSON)
|
||||||
|
}
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func listOrphanWeights(cmd *cobra.Command, c *ctlClient, node string, asJSON bool) error {
|
||||||
|
var found []adminapi.OrphanView
|
||||||
|
if err := c.get(cmd.Context(), "/api/v1/weights/orphans", &found); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if node != "" {
|
||||||
|
found = onlyNode(found, node)
|
||||||
|
}
|
||||||
|
if asJSON {
|
||||||
|
return emitJSON(found)
|
||||||
|
}
|
||||||
|
if len(found) == 0 {
|
||||||
|
fmt.Fprintln(cmd.OutOrStdout(),
|
||||||
|
"Nothing to reclaim: every file llmgw fetched is still wanted by a model.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
w := tab()
|
||||||
|
fmt.Fprintln(w, "NODE\tFILES\tWOULD FREE")
|
||||||
|
var total int64
|
||||||
|
for _, o := range found {
|
||||||
|
fmt.Fprintf(w, "%s\t%d\t%s\n", o.Node, len(o.Paths), fileSize(o.Bytes))
|
||||||
|
total += o.Bytes
|
||||||
|
}
|
||||||
|
_ = w.Flush()
|
||||||
|
|
||||||
|
for _, o := range found {
|
||||||
|
for _, p := range o.Paths {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "\n %s: %s", o.Node, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "\n\n%s in total. Delete with --collect.\n", fileSize(total))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func collectOrphans(cmd *cobra.Command, c *ctlClient, node string, asJSON bool) error {
|
||||||
|
body, err := json.Marshal(map[string]string{"node": node})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var results []struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Files int `json:"files"`
|
||||||
|
Bytes int64 `json:"bytes"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
if err := c.post(cmd.Context(), "/api/v1/weights/collect", body, &results); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if asJSON {
|
||||||
|
return emitJSON(results)
|
||||||
|
}
|
||||||
|
if len(results) == 0 {
|
||||||
|
fmt.Fprintln(cmd.OutOrStdout(), "Nothing to reclaim.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var freed int64
|
||||||
|
for _, r := range results {
|
||||||
|
if r.Error != "" {
|
||||||
|
// Reported rather than fatal: the other nodes' space is still worth
|
||||||
|
// freeing, and this one will be offered again next time.
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "%s: %s\n", r.Node, r.Error)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "%s: asked to remove %d files, %s\n",
|
||||||
|
r.Node, r.Files, fileSize(r.Bytes))
|
||||||
|
freed += r.Bytes
|
||||||
|
}
|
||||||
|
// "asked", not "removed": each node decides, and one that refuses a path it
|
||||||
|
// cannot prove it fetched is the feature working rather than failing.
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "\n%s requested. The nodes verify each path before deleting it.\n",
|
||||||
|
fileSize(freed))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func onlyNode(in []adminapi.OrphanView, node string) []adminapi.OrphanView {
|
||||||
|
out := make([]adminapi.OrphanView, 0, len(in))
|
||||||
|
for _, o := range in {
|
||||||
|
if o.Node == node {
|
||||||
|
out = append(out, o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// fileSize renders a size the way someone reads one when deciding whether a
|
||||||
|
// file is worth deleting.
|
||||||
|
//
|
||||||
|
// Separate from the per-token cache size renderer in modelplan, which never
|
||||||
|
// leaves KiB and MiB. Model files are tens of gigabytes, and "8601.6 MiB" is
|
||||||
|
// not a number anyone weighs a decision against.
|
||||||
|
func fileSize(b int64) string {
|
||||||
|
switch {
|
||||||
|
case b >= 1<<30:
|
||||||
|
return fmt.Sprintf("%.1f GB", float64(b)/(1<<30))
|
||||||
|
case b >= 1<<20:
|
||||||
|
return fmt.Sprintf("%.0f MB", float64(b)/(1<<20))
|
||||||
|
case b > 0:
|
||||||
|
return fmt.Sprintf("%d B", b)
|
||||||
|
default:
|
||||||
|
// The node reported no size. Saying "0 B" would suggest deleting it
|
||||||
|
// gains nothing, which is a different claim from not knowing.
|
||||||
|
return "size unknown"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTheForwardingDispatcherCarriesEveryCapability(t *testing.T) {
|
||||||
|
// dispatcherRef exists to break a construction cycle, and it satisfies the
|
||||||
|
// dispatcher interface by forwarding each method by hand. That makes it a
|
||||||
|
// place where capabilities go missing silently: the controller reaches
|
||||||
|
// past the interface by type assertion, and an assertion against this type
|
||||||
|
// fails for anything nobody remembered to forward.
|
||||||
|
//
|
||||||
|
// It happened. `ctl weights --collect` returned "this control plane cannot
|
||||||
|
// ask nodes to delete files" against a real cluster, while every unit test
|
||||||
|
// passed — because the test dispatchers do not implement the capability
|
||||||
|
// either, so its absence looked exactly like the expected case.
|
||||||
|
var ref any = &dispatcherRef{}
|
||||||
|
if _, ok := ref.(control.WeightRemover); !ok {
|
||||||
|
t.Error("dispatcherRef drops WeightRemover, so the control plane cannot reclaim disk")
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the thing it forwards to must have it, or the forwarding has nothing
|
||||||
|
// to reach.
|
||||||
|
var svc any = &agentsvc.Service{}
|
||||||
|
if _, ok := svc.(control.WeightRemover); !ok {
|
||||||
|
t.Error("the agent service does not implement WeightRemover")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,251 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newEngineCheckCmd() *cobra.Command {
|
||||||
|
var (
|
||||||
|
help string
|
||||||
|
bin string
|
||||||
|
nodes int
|
||||||
|
tp int
|
||||||
|
perNode int
|
||||||
|
)
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "engine-check <engine>",
|
||||||
|
Short: "Check that an engine still accepts the flags llmgw builds for it",
|
||||||
|
Long: "llmgw's launch arguments are read from each engine's documentation, and\n" +
|
||||||
|
"engines rename and remove flags between versions. When that happens the\n" +
|
||||||
|
"failure arrives minutes into a model load, in an engine's own error\n" +
|
||||||
|
"message, on a node nobody was watching.\n\n" +
|
||||||
|
"This asks the engine itself. It runs the engine's --help and checks every\n" +
|
||||||
|
"flag llmgw would pass against it, which needs no GPU — a CLI prints its\n" +
|
||||||
|
"help on any machine. Run it after upgrading an engine.\n\n" +
|
||||||
|
"With --help-output it reads a saved --help instead of running anything,\n" +
|
||||||
|
"so the check works from a laptop against help captured on a node.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
name := args[0]
|
||||||
|
reg := engines.NewRegistry(engines.Config{})
|
||||||
|
eng, err := reg.Get(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// The binary lives where the agent's configuration says, which is
|
||||||
|
// rarely on PATH: a check that cannot find the engine reports
|
||||||
|
// nothing rather than reporting a problem.
|
||||||
|
cfg := engines.Config{}
|
||||||
|
if bin != "" {
|
||||||
|
switch name {
|
||||||
|
case "llamacpp":
|
||||||
|
cfg.LlamaCpp.Bin = bin
|
||||||
|
case "vllm":
|
||||||
|
cfg.VLLM.Bin = bin
|
||||||
|
case "sglang":
|
||||||
|
cfg.SGLang.Python = bin
|
||||||
|
case "sim":
|
||||||
|
cfg.Sim.Bin = bin
|
||||||
|
}
|
||||||
|
reg = engines.NewRegistry(cfg)
|
||||||
|
if eng, err = reg.Get(name); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
spec := representativeShard(name, nodes, tp, perNode)
|
||||||
|
cmds, err := eng.Commands(spec)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("building a command for %s: %w", name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
out, errOut := cmd.OutOrStdout(), cmd.ErrOrStderr()
|
||||||
|
var checked, skipped int
|
||||||
|
var unknown []string
|
||||||
|
|
||||||
|
// Per command, and that distinction is the whole point. A shard can
|
||||||
|
// be several processes — vLLM under Ray is `ray start` and then
|
||||||
|
// `vllm serve` — and they are different programs with different
|
||||||
|
// vocabularies. Checking the sidecar's flags against the engine's
|
||||||
|
// help reports --head and --num-gpus as missing, which is true of
|
||||||
|
// vllm and irrelevant, because ray is what receives them.
|
||||||
|
for i, c := range cmds {
|
||||||
|
fmt.Fprintf(errOut, " %s %s\n", c.Path, strings.Join(c.Args, " "))
|
||||||
|
|
||||||
|
text := ""
|
||||||
|
switch {
|
||||||
|
case help != "" && i == len(cmds)-1:
|
||||||
|
// A supplied help belongs to the engine, which is the last
|
||||||
|
// command; the sidecars have their own and we do not have it.
|
||||||
|
text = help
|
||||||
|
case help != "":
|
||||||
|
skipped++
|
||||||
|
fmt.Fprintf(errOut, " skipped: --help-output describes the engine, not %s\n", c.Name)
|
||||||
|
continue
|
||||||
|
default:
|
||||||
|
var err error
|
||||||
|
if text, err = runHelp(cmd.Context(), c); err != nil {
|
||||||
|
skipped++
|
||||||
|
fmt.Fprintf(errOut, " skipped: %v\n", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A truncated help produces confident nonsense: every flag below
|
||||||
|
// the cut looks missing. Real help documents --help itself,
|
||||||
|
// which is the cheapest signal that we have all of it.
|
||||||
|
if !mentions(text, "--help") {
|
||||||
|
fmt.Fprintf(errOut, " warning: this help does not document --help, "+
|
||||||
|
"which usually means it was truncated; flags below the cut will look missing\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, a := range c.Args {
|
||||||
|
flag, _, _ := strings.Cut(a, "=")
|
||||||
|
if !strings.HasPrefix(flag, "--") || seen[flag] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[flag] = true
|
||||||
|
checked++
|
||||||
|
if !mentions(text, flag) {
|
||||||
|
unknown = append(unknown, c.Name+": "+flag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(unknown)
|
||||||
|
|
||||||
|
fmt.Fprintf(out, "%s: %d flags checked", name, checked)
|
||||||
|
if skipped > 0 {
|
||||||
|
fmt.Fprintf(out, ", %d commands skipped", skipped)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out)
|
||||||
|
|
||||||
|
if len(unknown) == 0 {
|
||||||
|
if checked == 0 {
|
||||||
|
return fmt.Errorf("nothing was checked; install the engine here or pass --help-output")
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "every flag llmgw builds is accepted")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "\n%d flags not mentioned by the program they are passed to:\n", len(unknown))
|
||||||
|
for _, f := range unknown {
|
||||||
|
fmt.Fprintf(out, " %s\n", f)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%s: %d flags are not accepted", name, len(unknown))
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
f := cmd.Flags()
|
||||||
|
f.StringVar(&help, "help-output", "", "a saved --help to check against, instead of running the engine")
|
||||||
|
f.StringVar(&bin, "bin", "", "where the engine lives, when it is not on PATH")
|
||||||
|
f.IntVar(&nodes, "nodes", 2, "shape the command for a replica of this many nodes")
|
||||||
|
f.IntVar(&tp, "tp", 8, "tensor parallel degree")
|
||||||
|
f.IntVar(&perNode, "gpus-per-node", 4, "GPUs each node contributes")
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
// representativeShard is the widest command llmgw would ever build, because a
|
||||||
|
// flag that only appears in the multi-node case is exactly the one nobody
|
||||||
|
// notices has been renamed.
|
||||||
|
func representativeShard(engine string, nodes, tp, perNode int) engines.ShardSpec {
|
||||||
|
if nodes < 1 {
|
||||||
|
nodes = 1
|
||||||
|
}
|
||||||
|
return engines.ShardSpec{
|
||||||
|
ReplicaID: "check", ShardID: "check-r0", Rank: 0, NNodes: nodes,
|
||||||
|
LeaderAddr: "10.0.0.1", LeaderPort: 20001,
|
||||||
|
GPUIndices: seq(perNode), HCAs: []string{"mlx5_0"},
|
||||||
|
MgmtInterface: "eth0", HTTPPort: 20000,
|
||||||
|
Profile: engines.Profile{
|
||||||
|
Name: "check", Engine: engine, TP: tp, Nodes: nodes, GPUsPerNode: perNode,
|
||||||
|
ModelPath: "/models/x", ServedModelName: "x",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func seq(n int) []int {
|
||||||
|
out := make([]int, n)
|
||||||
|
for i := range out {
|
||||||
|
out[i] = i
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// helpArgs is how each program is asked for its *complete* help.
|
||||||
|
//
|
||||||
|
// Not every program gives it all up for --help. vLLM prints a summary listing
|
||||||
|
// section names and tells you to ask again per section; checking against that
|
||||||
|
// reports every flag as missing, confidently and wrongly. This is exactly what
|
||||||
|
// happened the first time this ran against a real vLLM.
|
||||||
|
var helpArgs = map[string][]string{
|
||||||
|
"vllm": {"--help=all"},
|
||||||
|
}
|
||||||
|
|
||||||
|
// runHelp asks one program to describe itself.
|
||||||
|
func runHelp(ctx context.Context, c engines.Command) (string, error) {
|
||||||
|
// Everything up to the first flag, plus the help request. Passing the real
|
||||||
|
// arguments would start the engine.
|
||||||
|
var args []string
|
||||||
|
for _, a := range c.Args {
|
||||||
|
if strings.HasPrefix(a, "-") {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
args = append(args, a)
|
||||||
|
}
|
||||||
|
ask, ok := helpArgs[filepath.Base(c.Path)]
|
||||||
|
if !ok {
|
||||||
|
ask = []string{"--help"}
|
||||||
|
}
|
||||||
|
args = append(args, ask...)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
ec := exec.CommandContext(ctx, c.Path, args...)
|
||||||
|
// vLLM refuses to do anything at all on a machine with no accelerator,
|
||||||
|
// including print its help, unless it is told what to assume. Asking an
|
||||||
|
// engine about its flags should not require the hardware it runs on.
|
||||||
|
ec.Env = append(ec.Environ(), "VLLM_TARGET_DEVICE=cpu")
|
||||||
|
|
||||||
|
raw, err := ec.CombinedOutput()
|
||||||
|
if err != nil && len(raw) == 0 {
|
||||||
|
return "", fmt.Errorf("could not run %s: %w (install it here, or pass --help-output)",
|
||||||
|
c.Path, err)
|
||||||
|
}
|
||||||
|
return string(raw), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// mentions reports whether the help text documents a flag.
|
||||||
|
//
|
||||||
|
// Matched on a word boundary, so --port does not satisfy --port-range and a
|
||||||
|
// missing flag cannot hide behind a longer one that happens to contain it.
|
||||||
|
func mentions(help, flag string) bool {
|
||||||
|
for i := 0; ; {
|
||||||
|
j := strings.Index(help[i:], flag)
|
||||||
|
if j < 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
end := i + j + len(flag)
|
||||||
|
if end == len(help) || !isFlagChar(help[end]) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
i = end
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func isFlagChar(b byte) bool {
|
||||||
|
return b == '-' || b == '_' || (b >= 'a' && b <= 'z') ||
|
||||||
|
(b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9')
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAFlagIsNotSatisfiedByALongerOne(t *testing.T) {
|
||||||
|
// The failure this guards against is silent: --port would be reported as
|
||||||
|
// present because --port-range is documented, and a renamed flag would sail
|
||||||
|
// through the check that exists to catch renamed flags.
|
||||||
|
help := " --port-range N a range\n --model-name S a name\n"
|
||||||
|
|
||||||
|
if mentions(help, "--port") {
|
||||||
|
t.Error("--port was satisfied by --port-range")
|
||||||
|
}
|
||||||
|
if mentions(help, "--model") {
|
||||||
|
t.Error("--model was satisfied by --model-name")
|
||||||
|
}
|
||||||
|
if !mentions(help, "--port-range") {
|
||||||
|
t.Error("--port-range was not found")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFlagsAreFoundInTheShapesHelpTextUses(t *testing.T) {
|
||||||
|
help := `
|
||||||
|
-m, --model FNAME model path
|
||||||
|
--host HOST ip address
|
||||||
|
--parallel, -np N slots
|
||||||
|
--metrics enable prometheus
|
||||||
|
--alias=NAME served name
|
||||||
|
`
|
||||||
|
for _, flag := range []string{"--model", "--host", "--parallel", "--metrics", "--alias"} {
|
||||||
|
if !mentions(help, flag) {
|
||||||
|
t.Errorf("%s was not found in a help text that documents it", flag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, flag := range []string{"--nonsense", "--mode", "--para"} {
|
||||||
|
if mentions(help, flag) {
|
||||||
|
t.Errorf("%s was found in a help text that does not document it", flag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheRepresentativeShardIsTheWidestOne(t *testing.T) {
|
||||||
|
// A flag that only appears in the multi-node case is exactly the one nobody
|
||||||
|
// notices has been renamed, so the command built for checking has to be the
|
||||||
|
// widest llmgw would ever produce.
|
||||||
|
spec := representativeShard("vllm", 2, 8, 4)
|
||||||
|
if spec.NNodes != 2 || spec.Profile.Nodes != 2 {
|
||||||
|
t.Errorf("not a multi-node shard: %+v", spec)
|
||||||
|
}
|
||||||
|
if len(spec.GPUIndices) != 4 {
|
||||||
|
t.Errorf("%d GPUs, want 4", len(spec.GPUIndices))
|
||||||
|
}
|
||||||
|
if spec.LeaderAddr == "" || spec.LeaderPort == 0 {
|
||||||
|
t.Error("no rendezvous point, so the flags that carry it will not appear")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestATruncatedHelpIsSuspected(t *testing.T) {
|
||||||
|
// Found by truncating llama.cpp's 725-line help to 400 lines and being told,
|
||||||
|
// confidently, that five flags the running cluster uses do not exist.
|
||||||
|
full := " --help show this\n --model F path\n"
|
||||||
|
if !mentions(full, "--help") {
|
||||||
|
t.Fatal("the guard's own signal is missing from a complete help")
|
||||||
|
}
|
||||||
|
truncated := " --model F path\n"
|
||||||
|
if mentions(truncated, "--help") {
|
||||||
|
t.Error("a truncated help was taken for a complete one")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,327 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base32"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"text/tabwriter"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newTenancyCmd() *cobra.Command {
|
||||||
|
var dbPath string
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "keys",
|
||||||
|
Short: "Manage tenants and API keys",
|
||||||
|
Long: "Works directly on the store file rather than through the admin API, so keys\n" +
|
||||||
|
"can be issued before the server is running and without exposing an endpoint\n" +
|
||||||
|
"that mints credentials.",
|
||||||
|
}
|
||||||
|
cmd.PersistentFlags().StringVar(&dbPath, "store", "/var/lib/llmgw/llmgw.db", "path to the store")
|
||||||
|
|
||||||
|
open := func(ctx context.Context) (*store.SQLite, error) { return store.OpenSQLite(ctx, dbPath) }
|
||||||
|
|
||||||
|
cmd.AddCommand(
|
||||||
|
newKeyInitCmd(open),
|
||||||
|
newKeyCreateCmd(open),
|
||||||
|
newKeyListCmd(open),
|
||||||
|
newKeyRevokeCmd(open),
|
||||||
|
newLimitsCmd(open),
|
||||||
|
newThreadsCmd(open),
|
||||||
|
newMemoryCmd(open),
|
||||||
|
)
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
type opener func(context.Context) (*store.SQLite, error)
|
||||||
|
|
||||||
|
func newKeyInitCmd(open opener) *cobra.Command {
|
||||||
|
var org, team, project string
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "init",
|
||||||
|
Short: "Create the tenancy tree a key hangs off",
|
||||||
|
Long: "Idempotent: running it again renames rather than duplicating, so it can be\n" +
|
||||||
|
"kept in whatever provisions the machine.",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
s, err := open(cmd.Context())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
ctx := cmd.Context()
|
||||||
|
if err := s.PutOrg(ctx, store.Org{ID: slug(org), Name: org}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := s.PutTeam(ctx, store.Team{ID: slug(team), OrgID: slug(org), Name: team}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := s.PutProject(ctx, store.Project{
|
||||||
|
ID: slug(project), TeamID: slug(team), Name: project,
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "%s / %s / %s\n", slug(org), slug(team), slug(project))
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cmd.Flags().StringVar(&org, "org", "default", "organization name")
|
||||||
|
cmd.Flags().StringVar(&team, "team", "default", "team name")
|
||||||
|
cmd.Flags().StringVar(&project, "project", "default", "project name")
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func newKeyCreateCmd(open opener) *cobra.Command {
|
||||||
|
var project, band, profile, baseURL string
|
||||||
|
var expires, claimTTL time.Duration
|
||||||
|
var claim bool
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "create <name>",
|
||||||
|
Short: "Issue an API key",
|
||||||
|
Long: "The key is printed once and never again: only its hash is stored, so there\n" +
|
||||||
|
"is nothing to recover it from. That is the point — a leaked database hands\n" +
|
||||||
|
"over no working credential.\n\n" +
|
||||||
|
"With --claim it prints a one-time link instead. Send the link rather than\n" +
|
||||||
|
"the key: a key pasted into a chat window stays in that chat window forever,\n" +
|
||||||
|
"while a link works once and then expires. The key is sealed with the link's\n" +
|
||||||
|
"own token, which is not stored, so a pending claim is not a usable\n" +
|
||||||
|
"credential sitting in the database either.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
s, err := open(cmd.Context())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
secret, k, err := store.NewKey(randomID("key"), args[0], project)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
k.CreatedAt = time.Now()
|
||||||
|
k.Band, k.RoutingProfile = band, profile
|
||||||
|
if expires > 0 {
|
||||||
|
k.ExpiresAt = k.CreatedAt.Add(expires)
|
||||||
|
}
|
||||||
|
if err := s.CreateKey(cmd.Context(), k); err != nil {
|
||||||
|
return fmt.Errorf("creating the key (does project %q exist? run `llmgw keys init`): %w",
|
||||||
|
project, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The one output that matters goes to stdout and everything else to
|
||||||
|
// stderr, so `keys create ... > out.txt` captures it and nothing
|
||||||
|
// else. Cobra's Print writes to stderr, which would have made the
|
||||||
|
// useful half the half that cannot be piped.
|
||||||
|
out := cmd.OutOrStdout()
|
||||||
|
|
||||||
|
if claim {
|
||||||
|
if baseURL == "" {
|
||||||
|
return errors.New("--claim needs --base-url, the address users reach this gateway at")
|
||||||
|
}
|
||||||
|
token, err := store.NewClaimToken()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := s.CreateClaim(cmd.Context(), k.ID, token, secret, claimTTL); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
link := strings.TrimRight(baseURL, "/") + "/claim/" + token
|
||||||
|
fmt.Fprintln(out, link)
|
||||||
|
cmd.PrintErrf("\nid %s\nproject %s\nexpires %s\n",
|
||||||
|
k.ID, k.ProjectID, time.Now().Add(orDefault(claimTTL, store.ClaimTTL)).Format(time.RFC3339))
|
||||||
|
cmd.PrintErrln("\nSend the link. It works once, and the key is shown only to whoever opens it.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintln(out, secret)
|
||||||
|
cmd.PrintErrf("\nid %s\nproject %s\n", k.ID, k.ProjectID)
|
||||||
|
if !k.ExpiresAt.IsZero() {
|
||||||
|
cmd.PrintErrf("expires %s\n", k.ExpiresAt.Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
cmd.PrintErrln("\nThis is the only time the key is shown. Store it now.")
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cmd.Flags().StringVar(&project, "project", "default", "project the key belongs to")
|
||||||
|
cmd.Flags().StringVar(&band, "band", "", "admission band; empty inherits")
|
||||||
|
cmd.Flags().StringVar(&profile, "profile", "", "routing profile; empty inherits")
|
||||||
|
cmd.Flags().DurationVar(&expires, "expires-in", 0, "lifetime; 0 never expires")
|
||||||
|
cmd.Flags().BoolVar(&claim, "claim", false, "print a one-time link instead of the key")
|
||||||
|
cmd.Flags().StringVar(&baseURL, "base-url", "", "public address of the gateway, for --claim")
|
||||||
|
cmd.Flags().DurationVar(&claimTTL, "claim-ttl", store.ClaimTTL, "how long the link stays usable")
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func orDefault(d, fallback time.Duration) time.Duration {
|
||||||
|
if d <= 0 {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
func newKeyListCmd(open opener) *cobra.Command {
|
||||||
|
var project string
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "list",
|
||||||
|
Short: "List a project's keys",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
s, err := open(cmd.Context())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
keys, err := s.Keys(cmd.Context(), project)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(keys) == 0 {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "no keys in project %q\n", project)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0)
|
||||||
|
fmt.Fprintln(w, "ID\tNAME\tKEY\tBAND\tSTATE\tLAST USED")
|
||||||
|
now := time.Now()
|
||||||
|
for _, k := range keys {
|
||||||
|
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||||
|
k.ID, k.Name, k.Prefix, dash(k.Band), keyState(k, now), ago(k.LastUsedAt, now))
|
||||||
|
}
|
||||||
|
return w.Flush()
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cmd.Flags().StringVar(&project, "project", "default", "project to list")
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func newKeyRevokeCmd(open opener) *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "revoke <key-id>",
|
||||||
|
Short: "Stop a key working, without deleting it",
|
||||||
|
Long: "A soft delete. Usage rows reference the key, and an audit trail that loses\n" +
|
||||||
|
"its subject is not an audit trail.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
s, err := open(cmd.Context())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
if err := s.RevokeKey(cmd.Context(), args[0], time.Now()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "revoked %s\n", args[0])
|
||||||
|
cmd.PrintErrln("A running server keeps accepting it until its cache expires (auth.cache_ttl).")
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newLimitsCmd(open opener) *cobra.Command {
|
||||||
|
var scope, id, period string
|
||||||
|
var rpm, tpm int
|
||||||
|
var budget int64
|
||||||
|
var models []string
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "limits",
|
||||||
|
Short: "Set limits at one level of the tenancy tree",
|
||||||
|
Long: "Limits compose downward by taking the most restrictive value, so a key\n" +
|
||||||
|
"cannot raise what its organization set. Model allowlists intersect: a key\n" +
|
||||||
|
"cannot add a model its team withheld.",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
s, err := open(cmd.Context())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
l := store.Limits{Scope: store.Scope(scope), ScopeID: id}
|
||||||
|
if cmd.Flags().Changed("rpm") {
|
||||||
|
l.RPM = &rpm
|
||||||
|
}
|
||||||
|
if cmd.Flags().Changed("tpm") {
|
||||||
|
l.TPM = &tpm
|
||||||
|
}
|
||||||
|
if cmd.Flags().Changed("budget") {
|
||||||
|
l.Budget = &budget
|
||||||
|
l.BudgetPeriod = store.Period(period)
|
||||||
|
}
|
||||||
|
if cmd.Flags().Changed("models") {
|
||||||
|
// An explicitly empty list is an allowlist that permits
|
||||||
|
// nothing, which is different from setting no list at all.
|
||||||
|
if models == nil {
|
||||||
|
models = []string{}
|
||||||
|
}
|
||||||
|
l.AllowedModels = models
|
||||||
|
}
|
||||||
|
if err := s.PutLimits(cmd.Context(), l); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "limits set on %s %s\n", scope, id)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cmd.Flags().StringVar(&scope, "scope", "project", "org | team | project | key")
|
||||||
|
cmd.Flags().StringVar(&id, "id", "default", "id at that scope")
|
||||||
|
cmd.Flags().IntVar(&rpm, "rpm", 0, "requests per minute")
|
||||||
|
cmd.Flags().IntVar(&tpm, "tpm", 0, "tokens per minute")
|
||||||
|
cmd.Flags().Int64Var(&budget, "budget", 0, "budget per period, in micro-units")
|
||||||
|
cmd.Flags().StringVar(&period, "period", "month", "day | month")
|
||||||
|
cmd.Flags().StringSliceVar(&models, "models", nil, "allowed models; empty allows none")
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func keyState(k store.Key, now time.Time) string {
|
||||||
|
switch {
|
||||||
|
case !k.RevokedAt.IsZero():
|
||||||
|
return "revoked"
|
||||||
|
case !k.ExpiresAt.IsZero() && !now.Before(k.ExpiresAt):
|
||||||
|
return "expired"
|
||||||
|
default:
|
||||||
|
return "active"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func ago(t, now time.Time) string {
|
||||||
|
if t.IsZero() {
|
||||||
|
return "never"
|
||||||
|
}
|
||||||
|
return now.Sub(t).Round(time.Second).String() + " ago"
|
||||||
|
}
|
||||||
|
|
||||||
|
// slug turns a human name into an id.
|
||||||
|
func slug(name string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, r := range strings.ToLower(strings.TrimSpace(name)) {
|
||||||
|
switch {
|
||||||
|
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||||
|
b.WriteRune(r)
|
||||||
|
case r == ' ' || r == '-' || r == '_':
|
||||||
|
b.WriteByte('-')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if b.Len() == 0 {
|
||||||
|
return "default"
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func randomID(prefix string) string {
|
||||||
|
var buf [10]byte
|
||||||
|
_, _ = rand.Read(buf[:])
|
||||||
|
return prefix + "-" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(buf[:]))
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
// newMemoryCmd turns note storage on for a project, or off.
|
||||||
|
//
|
||||||
|
// A command rather than a configuration key for the same reason threads are:
|
||||||
|
// keeping what an agent decided to remember is a decision about data, not about
|
||||||
|
// deployment, and it should look like an act rather than a default nobody read.
|
||||||
|
func newMemoryCmd(open opener) *cobra.Command {
|
||||||
|
var (
|
||||||
|
project string
|
||||||
|
hours int
|
||||||
|
off bool
|
||||||
|
)
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "memory",
|
||||||
|
Short: "Keep notes an agent writes for itself, across conversations",
|
||||||
|
Long: "With this on, an agent can save and search notes through /v1/memory or the\n" +
|
||||||
|
"MCP tools at /mcp (remember, recall, forget), and find them again in later\n" +
|
||||||
|
"conversations.\n\n" +
|
||||||
|
"Off for every project until this is run. The retention is required: notes\n" +
|
||||||
|
"expire that many hours after they were last written, so a store nobody\n" +
|
||||||
|
"chose a number for does not keep everything forever.",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
if !off && hours <= 0 {
|
||||||
|
return fmt.Errorf("--retain-hours is required: say how long notes are kept, or pass --off")
|
||||||
|
}
|
||||||
|
s, err := open(cmd.Context())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
keep := hours
|
||||||
|
if off {
|
||||||
|
keep = 0
|
||||||
|
}
|
||||||
|
if err := s.SetMemoryRetention(cmd.Context(), project, keep); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if keep == 0 {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(),
|
||||||
|
"%s keeps no notes. Existing ones are removed at the next prune.\n", project)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(),
|
||||||
|
"%s keeps notes for %d hours after they were last written.\n", project, keep)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cmd.Flags().StringVar(&project, "project", "default", "project id")
|
||||||
|
cmd.Flags().IntVar(&hours, "retain-hours", 0, "how long a note is kept after it was last written")
|
||||||
|
cmd.Flags().BoolVar(&off, "off", false, "stop keeping notes for this project")
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os/user"
|
||||||
|
|
||||||
|
"github.com/knadh/koanf/parsers/yaml"
|
||||||
|
"github.com/knadh/koanf/providers/file"
|
||||||
|
"github.com/knadh/koanf/v2"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// newModelCmd adds and removes models without a restart.
|
||||||
|
//
|
||||||
|
// Adding a model used to mean editing a file on the control plane and
|
||||||
|
// restarting it: a settling window, adoption churn, and on one occasion two
|
||||||
|
// minutes of outage. For a fixed production set that is a reasonable trade; for
|
||||||
|
// a platform whose promise is that people can try things, it is the difference
|
||||||
|
// between trying one and not bothering.
|
||||||
|
func newModelCmd(client func() *ctlClient) *cobra.Command {
|
||||||
|
c := &cobra.Command{
|
||||||
|
Use: "model",
|
||||||
|
Short: "Add or remove models while the cluster is running",
|
||||||
|
}
|
||||||
|
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client))
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func newModelAddCmd(client func() *ctlClient) *cobra.Command {
|
||||||
|
var path string
|
||||||
|
c := &cobra.Command{
|
||||||
|
Use: "add -f <file.yaml>",
|
||||||
|
Short: "Add a model from a definition file",
|
||||||
|
Long: "The definition is the same YAML a model takes in server.yaml, so an\n" +
|
||||||
|
"entry can be moved between the two by copying it.\n\n" +
|
||||||
|
"The catalog is rebuilt and validated before anything is written, so a\n" +
|
||||||
|
"definition that collides with an existing name, or names a fallback that\n" +
|
||||||
|
"does not exist, is refused rather than stored and then ignored on every\n" +
|
||||||
|
"start. Once accepted it takes effect on the reconciler's next pass — no\n" +
|
||||||
|
"restart — and survives one.",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
if path == "" {
|
||||||
|
return fmt.Errorf("--file is required: the model definition, as YAML")
|
||||||
|
}
|
||||||
|
// Read by koanf, the same parser and the same tags that read
|
||||||
|
// server.yaml — so an entry can be moved between the two by copying
|
||||||
|
// it, and a field that works in one cannot silently mean nothing in
|
||||||
|
// the other.
|
||||||
|
k := koanf.New(".")
|
||||||
|
if err := k.Load(file.Provider(path), yaml.Parser()); err != nil {
|
||||||
|
return fmt.Errorf("%s: %w", path, err)
|
||||||
|
}
|
||||||
|
var m catalog.Model
|
||||||
|
if err := k.UnmarshalWithConf("", &m, koanf.UnmarshalConf{Tag: "koanf"}); err != nil {
|
||||||
|
return fmt.Errorf("%s: %w", path, err)
|
||||||
|
}
|
||||||
|
if m.Name == "" {
|
||||||
|
return fmt.Errorf("%s: the definition has no name", path)
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(m)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var res map[string]string
|
||||||
|
if err := client().post(cmd.Context(), "/api/v1/models", body, &res); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(),
|
||||||
|
"%s added. It will be placed on the next reconcile pass; watch it with:\n"+
|
||||||
|
" llmgw ctl models\n", m.Name)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c.Flags().StringVarP(&path, "file", "f", "", "YAML file holding the model definition")
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func newModelRemoveCmd(client func() *ctlClient) *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "remove <name>",
|
||||||
|
Short: "Remove a model that was added at runtime",
|
||||||
|
Long: "Only models added at runtime. One declared in server.yaml would come\n" +
|
||||||
|
"back on the next restart, so removing it here would leave the cluster\n" +
|
||||||
|
"disagreeing with its own configuration until then.\n\n" +
|
||||||
|
"Its replicas are stopped on the next pass, after the endpoint has been\n" +
|
||||||
|
"withdrawn, so traffic stops before the engine does.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
var res map[string]string
|
||||||
|
if err := client().delete(cmd.Context(), "/api/v1/models/"+args[0], &res); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "%s removed.\n", args[0])
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// actor names who made a change, for the audit trail on a stored model.
|
||||||
|
func actor() string {
|
||||||
|
if u, err := user.Current(); err == nil && u.Username != "" {
|
||||||
|
return u.Username
|
||||||
|
}
|
||||||
|
return "unknown"
|
||||||
|
}
|
||||||
@@ -0,0 +1,240 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||||
|
)
|
||||||
|
|
||||||
|
// newModelPlanCmd answers whether a model fits this cluster, and how.
|
||||||
|
//
|
||||||
|
// The arithmetic it replaces is the last thing standing between having GPUs and
|
||||||
|
// serving a model on them. Deciding tp, gpu_class and a context that will
|
||||||
|
// actually fit means knowing the model's layer count, its KV head count, what
|
||||||
|
// its weights weigh at a chosen quantisation, and how much of a card the
|
||||||
|
// runtime keeps — and getting it wrong is not an error message, it is an
|
||||||
|
// out-of-memory twenty minutes into a load.
|
||||||
|
//
|
||||||
|
// It reads the model from its own repository and the hardware from the running
|
||||||
|
// fleet, so neither half is a number somebody retyped.
|
||||||
|
func newModelPlanCmd(client func() *ctlClient) *cobra.Command {
|
||||||
|
var (
|
||||||
|
context int
|
||||||
|
params int64
|
||||||
|
precs []string
|
||||||
|
)
|
||||||
|
c := &cobra.Command{
|
||||||
|
Use: "plan <huggingface-repo>",
|
||||||
|
Short: "Work out whether a model fits this cluster, and how it should be split",
|
||||||
|
Long: "Reads the model's architecture from its repository and the hardware from\n" +
|
||||||
|
"the running fleet, then prints every way it could run — fewest GPUs\n" +
|
||||||
|
"first — and a catalog entry for the best one.\n\n" +
|
||||||
|
"Every candidate is listed rather than one answer, because the choice is\n" +
|
||||||
|
"not the planner's to make: with cards to spare you want the\n" +
|
||||||
|
"full-precision fit, and with one card free you want to know a quantised\n" +
|
||||||
|
"one exists.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
repo := args[0]
|
||||||
|
m, err := modelplan.FetchModel(cmd.Context(), repo)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if params > 0 {
|
||||||
|
m.Params = params
|
||||||
|
}
|
||||||
|
if m.Params == 0 {
|
||||||
|
return fmt.Errorf("%s does not publish a parameter count; pass --params", repo)
|
||||||
|
}
|
||||||
|
|
||||||
|
var chosen []modelplan.Precision
|
||||||
|
for _, name := range precs {
|
||||||
|
p, err := modelplan.PrecisionByName(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
chosen = append(chosen, p)
|
||||||
|
}
|
||||||
|
|
||||||
|
var nodes []adminapi.NodeView
|
||||||
|
if err := client().get(cmd.Context(), "/api/v1/nodes", &nodes); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
hw := hardwareFrom(nodes)
|
||||||
|
if len(hw) == 0 {
|
||||||
|
return fmt.Errorf("no node is ready, so there is nothing to plan against")
|
||||||
|
}
|
||||||
|
|
||||||
|
out := cmd.OutOrStdout()
|
||||||
|
fmt.Fprintf(out, "%s: %.1fB parameters, %d layers, %d KV heads, trained to %d tokens\n",
|
||||||
|
m.Name, float64(m.Params)/1e9, m.Layers, m.KVHeads, m.MaxContext)
|
||||||
|
|
||||||
|
want := context
|
||||||
|
if want <= 0 {
|
||||||
|
want = m.MaxContext
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "Planned for a %d-token pool. %s a token of context.\n\n",
|
||||||
|
want, humanBytes(m.KVBytesPerToken(modelplan.BF16)))
|
||||||
|
|
||||||
|
fits := modelplan.Fits(m, hw, want, chosen)
|
||||||
|
if len(fits) == 0 {
|
||||||
|
fmt.Fprintln(out, "It does not fit this cluster at any offered precision.")
|
||||||
|
fmt.Fprintln(out, "\nWhat would change that:")
|
||||||
|
fmt.Fprintln(out, " - a smaller context pool, with --context")
|
||||||
|
fmt.Fprintln(out, " - a heavier quantisation, with --precision q4_k_m")
|
||||||
|
fmt.Fprintln(out, " - more cards free; llmgw ctl status shows what is in use")
|
||||||
|
return errDoesNotFit
|
||||||
|
}
|
||||||
|
|
||||||
|
w := tab()
|
||||||
|
fmt.Fprintln(w, " GPUS\tPRECISION\tPER CARD\tFREE\tWHERE")
|
||||||
|
for _, f := range fits {
|
||||||
|
where := f.Class
|
||||||
|
if f.CPUOnly {
|
||||||
|
where = "cpu"
|
||||||
|
} else if f.Nodes > 1 {
|
||||||
|
where = fmt.Sprintf("%s, %d nodes on %s", f.Class, f.Nodes, f.Fabric)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, " %d\t%s\t%.1f GB\t%.1f GB\t%s\n",
|
||||||
|
f.TP, f.Precision.Name, f.PerGPUGB, f.Headroom, where)
|
||||||
|
}
|
||||||
|
_ = w.Flush()
|
||||||
|
|
||||||
|
fmt.Fprintf(out, "\nA catalog entry for the first of those:\n\n%s\n",
|
||||||
|
catalogEntry(m, fits[0], repo, want))
|
||||||
|
fmt.Fprintln(out, "Save it and add it with: llmgw ctl model add -f <file>")
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c.Flags().IntVar(&context, "context", 0, "token pool to plan for (default: the model's trained context)")
|
||||||
|
c.Flags().Int64Var(¶ms, "params", 0, "parameter count, when the repository does not publish one")
|
||||||
|
c.Flags().StringSliceVar(&precs, "precision", nil, "precisions to consider (default: bf16, fp8, q8_0, q4_k_m)")
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// errDoesNotFit gives a script something to test without parsing prose.
|
||||||
|
var errDoesNotFit = fmt.Errorf("the model does not fit this cluster")
|
||||||
|
|
||||||
|
// hardwareFrom turns the fleet into pools a replica could actually occupy.
|
||||||
|
//
|
||||||
|
// Grouped by class and fabric together, because a replica may never span two
|
||||||
|
// fabrics: eight free cards split across two fabrics are not eight cards for a
|
||||||
|
// model that needs eight, and treating them as one pool would plan something
|
||||||
|
// that hangs on its first all-reduce.
|
||||||
|
func hardwareFrom(nodes []adminapi.NodeView) []modelplan.Hardware {
|
||||||
|
type key struct{ class, fabric string }
|
||||||
|
pools := map[key]*modelplan.Hardware{}
|
||||||
|
|
||||||
|
for _, n := range nodes {
|
||||||
|
if n.State != "ready" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fabric := ""
|
||||||
|
if len(n.Fabrics) > 0 {
|
||||||
|
fabric = n.Fabrics[0]
|
||||||
|
}
|
||||||
|
class := ""
|
||||||
|
if len(n.Classes) > 0 {
|
||||||
|
class = n.Classes[0]
|
||||||
|
}
|
||||||
|
k := key{class, fabric}
|
||||||
|
if pools[k] == nil {
|
||||||
|
pools[k] = &modelplan.Hardware{Class: class, Fabric: fabric}
|
||||||
|
}
|
||||||
|
p := pools[k]
|
||||||
|
p.Nodes++
|
||||||
|
// The widest a single node could contribute, not the total: a tensor
|
||||||
|
// group takes the same number of cards from every node it spans.
|
||||||
|
if n.GPUFree > p.GPUsPerNode {
|
||||||
|
p.GPUsPerNode = n.GPUFree
|
||||||
|
}
|
||||||
|
if n.MemoryGB > p.HostMemGB {
|
||||||
|
p.HostMemGB = n.MemoryGB
|
||||||
|
}
|
||||||
|
// Card memory from the class label, which carries it by construction:
|
||||||
|
// "h100-80g" is eighty gigabytes and says so.
|
||||||
|
if class != "" && p.GPUMemGB == 0 {
|
||||||
|
p.GPUMemGB = memFromClass(class)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]modelplan.Hardware, 0, len(pools))
|
||||||
|
for _, p := range pools {
|
||||||
|
if p.Class != "" && (p.GPUMemGB == 0 || p.GPUsPerNode == 0) {
|
||||||
|
continue // a GPU pool with no free cards is not a pool
|
||||||
|
}
|
||||||
|
out = append(out, *p)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// memFromClass reads the card size out of a class label such as "h100-80g".
|
||||||
|
func memFromClass(class string) int {
|
||||||
|
i := strings.LastIndex(class, "-")
|
||||||
|
if i < 0 || !strings.HasSuffix(class, "g") {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
var gb int
|
||||||
|
if _, err := fmt.Sscanf(class[i+1:], "%dg", &gb); err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return gb
|
||||||
|
}
|
||||||
|
|
||||||
|
// catalogEntry renders a fit as the YAML that would serve it.
|
||||||
|
func catalogEntry(m modelplan.Model, f modelplan.Fit, source string, context int) string {
|
||||||
|
var b strings.Builder
|
||||||
|
name := strings.ToLower(m.Name)
|
||||||
|
fmt.Fprintf(&b, "name: %s\n", name)
|
||||||
|
if f.Precision.GGUF {
|
||||||
|
// A GGUF quantisation lives in a file, not a repository, and llama.cpp
|
||||||
|
// is given a path. Printing the repository id here would produce an
|
||||||
|
// entry that looks right and cannot load — which is the failure this
|
||||||
|
// whole command exists to prevent.
|
||||||
|
fmt.Fprintf(&b, "# %s is a GGUF quantisation: download the %s file and give its path.\n",
|
||||||
|
f.Precision.Name, f.Precision.Name)
|
||||||
|
fmt.Fprintf(&b, "source: /var/lib/llmgw-models/%s-%s.gguf\n", name, f.Precision.Name)
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(&b, "source: %s\n", source)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, "routing_profile: chat\n")
|
||||||
|
fmt.Fprintf(&b, "context_length: %d\n", context)
|
||||||
|
fmt.Fprintf(&b, "scaling: { min: 1, max: 1 }\n")
|
||||||
|
fmt.Fprintf(&b, "profiles:\n")
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case f.CPUOnly:
|
||||||
|
fmt.Fprintf(&b, " - name: cpu\n engine: %s\n", f.Precision.Engine)
|
||||||
|
fmt.Fprintf(&b, " min_memory_gb: %d\n", int(f.PerGPUGB)+2)
|
||||||
|
default:
|
||||||
|
fmt.Fprintf(&b, " - name: tp%d\n engine: %s\n", f.TP, f.Precision.Engine)
|
||||||
|
fmt.Fprintf(&b, " gpu_class: %s\n tp: %d\n", f.Class, f.TP)
|
||||||
|
if f.Nodes > 1 {
|
||||||
|
fmt.Fprintf(&b, " nodes: %d\n gpus_per_node: %d\n", f.Nodes, f.GPUsPerNode)
|
||||||
|
}
|
||||||
|
if q := f.Precision.Quantization; q != "" {
|
||||||
|
fmt.Fprintf(&b, " args: [\"--quantization\", \"%s\"]\n", q)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, " priority: 100\n weight: 1\n")
|
||||||
|
// Measured rather than guessed is the rule everywhere else in this system,
|
||||||
|
// and a planner should say so rather than pretend its estimate is one.
|
||||||
|
fmt.Fprintf(&b, " startup_timeout: 10m\n")
|
||||||
|
b.WriteString("\n# weight should come from `llmgw ctl bench` once it is running;\n")
|
||||||
|
b.WriteString("# 1 is a placeholder, not a measurement.\n")
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// humanBytes renders a per-token cache size, which is always small.
|
||||||
|
func humanBytes(n int64) string {
|
||||||
|
switch {
|
||||||
|
case n >= 1<<20:
|
||||||
|
return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20))
|
||||||
|
default:
|
||||||
|
return fmt.Sprintf("%.0f KiB", float64(n)/1024)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -40,6 +40,8 @@ func Root() *cobra.Command {
|
|||||||
newServerCmd(),
|
newServerCmd(),
|
||||||
newAgentCmd(),
|
newAgentCmd(),
|
||||||
newCtlCmd(),
|
newCtlCmd(),
|
||||||
|
newTenancyCmd(),
|
||||||
|
newUsersCmd(),
|
||||||
newSimEngineCmd(),
|
newSimEngineCmd(),
|
||||||
)
|
)
|
||||||
return root
|
return root
|
||||||
|
|||||||
+298
-9
@@ -2,28 +2,43 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/tls"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
"golang.org/x/net/http2"
|
||||||
|
"golang.org/x/net/http2/h2c"
|
||||||
"golang.org/x/sync/errgroup"
|
"golang.org/x/sync/errgroup"
|
||||||
|
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fleetmetrics"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
|
||||||
)
|
)
|
||||||
|
|
||||||
func newServerCmd() *cobra.Command {
|
func newServerCmd() *cobra.Command {
|
||||||
var (
|
var (
|
||||||
configPath string
|
configPath string
|
||||||
listen string
|
listen string
|
||||||
adminListen string
|
adminListen string
|
||||||
endpoints []string
|
metricsListen string
|
||||||
|
endpoints []string
|
||||||
)
|
)
|
||||||
|
|
||||||
cmd := &cobra.Command{
|
cmd := &cobra.Command{
|
||||||
@@ -45,6 +60,9 @@ func newServerCmd() *cobra.Command {
|
|||||||
if adminListen != "" {
|
if adminListen != "" {
|
||||||
cfg.AdminListen = adminListen
|
cfg.AdminListen = adminListen
|
||||||
}
|
}
|
||||||
|
if metricsListen != "" {
|
||||||
|
cfg.MetricsListen = metricsListen
|
||||||
|
}
|
||||||
for _, spec := range endpoints {
|
for _, spec := range endpoints {
|
||||||
ep, err := parseEndpointFlag(spec)
|
ep, err := parseEndpointFlag(spec)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -55,8 +73,10 @@ func newServerCmd() *cobra.Command {
|
|||||||
if err := cfg.Validate(); err != nil {
|
if err := cfg.Validate(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if len(cfg.Endpoints) == 0 {
|
controlMode := cfg.ControlPlaneMode()
|
||||||
return errors.New("no endpoints configured; use --endpoint or a config file")
|
if !controlMode && len(cfg.Endpoints) == 0 {
|
||||||
|
return errors.New("nothing to serve: declare models with profiles, " +
|
||||||
|
"or give --endpoint for a router-only run")
|
||||||
}
|
}
|
||||||
|
|
||||||
rt, err := router.New(cfg, log)
|
rt, err := router.New(cfg, log)
|
||||||
@@ -70,10 +90,93 @@ func newServerCmd() *cobra.Command {
|
|||||||
rt.Start(ctx)
|
rt.Start(ctx)
|
||||||
defer rt.Stop()
|
defer rt.Stop()
|
||||||
|
|
||||||
adminHandler, err := rt.AdminHandler()
|
// The control plane replaces the static endpoint list: agents
|
||||||
|
// register, the reconciler places replicas, and Ready ones are
|
||||||
|
// published to the router (plan §6.4).
|
||||||
|
var agentHandler http.Handler
|
||||||
|
var agentPath string
|
||||||
|
var admin *control.Controller
|
||||||
|
if controlMode {
|
||||||
|
cat, err := cfg.Catalog()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
live := catalog.NewLive(cat)
|
||||||
|
// The router derives its own view, and it is empty until told.
|
||||||
|
// Told here rather than at construction so there is exactly one
|
||||||
|
// path by which it learns the catalog, used at startup and at
|
||||||
|
// every change after.
|
||||||
|
rt.SetCatalog(cat)
|
||||||
|
|
||||||
|
ctl, svc, err := startControlPlane(ctx, cfg, live, rt, cfg.AgentTLS.Enabled(), log)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer ctl.Close()
|
||||||
|
admin = ctl
|
||||||
|
agentPath, agentHandler = svc.Handler()
|
||||||
|
|
||||||
|
// Before AdminHandler, which is where the registry is built —
|
||||||
|
// registering after it silently publishes nothing, which is
|
||||||
|
// exactly what the first version of this did.
|
||||||
|
//
|
||||||
|
// On the same /metrics as the traffic figures: correlating
|
||||||
|
// "the cluster was full" with "requests were queueing" should
|
||||||
|
// not need two systems.
|
||||||
|
if err := rt.RegisterMetrics(fleetmetrics.New(ctl)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// So a request for a model that is configured but not running
|
||||||
|
// starts it, rather than being told nothing serves it.
|
||||||
|
rt.SetActivator(ctl.Reconciler())
|
||||||
|
|
||||||
|
log.Info("control plane running",
|
||||||
|
"models", len(cfg.Models), "agent_listen", cfg.AgentListen,
|
||||||
|
"agent_mtls", cfg.AgentTLS.Enabled())
|
||||||
|
}
|
||||||
|
|
||||||
|
adminMux, err := rt.AdminHandler()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if admin != nil {
|
||||||
|
api := adminapi.New(admin)
|
||||||
|
// Catalog changes need somewhere to persist, so this is wired
|
||||||
|
// only when there is a store. Without one, the endpoints say so
|
||||||
|
// rather than accepting a model that vanishes on restart.
|
||||||
|
if st := rt.Store(); st != nil {
|
||||||
|
svc := modelsvc.New(cfg.Models, st, admin, log)
|
||||||
|
// Rebuild the catalog from configuration plus what was
|
||||||
|
// stored, and put that in force — otherwise models added
|
||||||
|
// before the last restart would be silently absent until
|
||||||
|
// someone added them again.
|
||||||
|
if built, err := svc.Load(ctx); err != nil {
|
||||||
|
log.Error("could not load stored models; continuing with configuration only", "err", err)
|
||||||
|
} else {
|
||||||
|
admin.ReplaceCatalog(ctx, built)
|
||||||
|
}
|
||||||
|
api.SetModels(svc)
|
||||||
|
}
|
||||||
|
api.Register(adminMux)
|
||||||
|
|
||||||
|
// The dashboard lives here and only here. This listener is
|
||||||
|
// bound to loopback by default and also serves pprof, which can
|
||||||
|
// dump the process heap — where API keys sit in plaintext.
|
||||||
|
// Reaching the page from elsewhere is an SSH tunnel's job:
|
||||||
|
// ssh -L 9090:127.0.0.1:9090 <host>
|
||||||
|
var usage web.UsageSource
|
||||||
|
if st := rt.Store(); st != nil {
|
||||||
|
usage = st
|
||||||
|
}
|
||||||
|
adminMux.HandleFunc("GET /", web.Dashboard(
|
||||||
|
api, usage, version.Version, 5*time.Second, true, log))
|
||||||
|
adminMux.Handle("GET /_static/", web.Static())
|
||||||
|
|
||||||
|
// The same page on the client-facing port at /admin, behind a
|
||||||
|
// session and an admin flag. Handing the router the data rather
|
||||||
|
// than the port keeps pprof where it is.
|
||||||
|
rt.SetClusterView(api, usage)
|
||||||
|
}
|
||||||
|
|
||||||
apiSrv := &http.Server{
|
apiSrv := &http.Server{
|
||||||
Handler: rt.Handler(),
|
Handler: rt.Handler(),
|
||||||
@@ -82,7 +185,7 @@ func newServerCmd() *cobra.Command {
|
|||||||
// enforces its own per-phase deadlines.
|
// enforces its own per-phase deadlines.
|
||||||
}
|
}
|
||||||
adminSrv := &http.Server{
|
adminSrv := &http.Server{
|
||||||
Handler: adminHandler,
|
Handler: adminMux,
|
||||||
ReadHeaderTimeout: 10 * time.Second,
|
ReadHeaderTimeout: 10 * time.Second,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -96,21 +199,92 @@ func newServerCmd() *cobra.Command {
|
|||||||
return fmt.Errorf("listening on %s: %w", cfg.AdminListen, err)
|
return fmt.Errorf("listening on %s: %w", cfg.AdminListen, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A metrics-only listener, when one is configured. Started after
|
||||||
|
// AdminHandler, which is where the registry it serves is built.
|
||||||
|
var metricsSrv *http.Server
|
||||||
|
var metricsLn net.Listener
|
||||||
|
if cfg.MetricsListen != "" {
|
||||||
|
h := rt.MetricsHandler()
|
||||||
|
if h == nil {
|
||||||
|
_ = apiLn.Close()
|
||||||
|
_ = adminLn.Close()
|
||||||
|
return fmt.Errorf("metrics handler is not ready; this is a wiring fault, not a configuration one")
|
||||||
|
}
|
||||||
|
metricsLn, err = net.Listen("tcp", cfg.MetricsListen)
|
||||||
|
if err != nil {
|
||||||
|
_ = apiLn.Close()
|
||||||
|
_ = adminLn.Close()
|
||||||
|
return fmt.Errorf("listening on %s: %w", cfg.MetricsListen, err)
|
||||||
|
}
|
||||||
|
metricsSrv = &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second}
|
||||||
|
go func() {
|
||||||
|
if err := metricsSrv.Serve(metricsLn); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||||
|
log.Error("metrics listener stopped", "err", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
log.Info("router listening",
|
log.Info("router listening",
|
||||||
"api", apiLn.Addr().String(),
|
"api", apiLn.Addr().String(),
|
||||||
"admin", adminLn.Addr().String(),
|
"admin", adminLn.Addr().String(),
|
||||||
|
"metrics", metricsAddr(metricsLn),
|
||||||
"endpoints", len(cfg.Endpoints),
|
"endpoints", len(cfg.Endpoints),
|
||||||
"default_profile", cfg.Router.DefaultProfile,
|
"default_profile", cfg.Router.DefaultProfile,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var agentSrv *http.Server
|
||||||
|
var agentLn net.Listener
|
||||||
|
if agentHandler != nil {
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
mux.Handle(agentPath, agentHandler)
|
||||||
|
agentSrv = &http.Server{
|
||||||
|
Handler: mux,
|
||||||
|
ReadHeaderTimeout: 10 * time.Second,
|
||||||
|
}
|
||||||
|
if !cfg.AgentTLS.Enabled() {
|
||||||
|
// Without TLS, agents speak plaintext HTTP/2, which Go will
|
||||||
|
// not do unless asked. Nothing then stops a node from
|
||||||
|
// claiming to be another one, so this belongs on a trusted
|
||||||
|
// network only.
|
||||||
|
agentSrv.Handler = h2c.NewHandler(mux, &http2.Server{})
|
||||||
|
}
|
||||||
|
if agentLn, err = net.Listen("tcp", cfg.AgentListen); err != nil {
|
||||||
|
_ = apiLn.Close()
|
||||||
|
_ = adminLn.Close()
|
||||||
|
return fmt.Errorf("listening on %s: %w", cfg.AgentListen, err)
|
||||||
|
}
|
||||||
|
if cfg.AgentTLS.Enabled() {
|
||||||
|
tc, terr := cfg.AgentTLS.ServerConfig()
|
||||||
|
if terr != nil {
|
||||||
|
_ = apiLn.Close()
|
||||||
|
_ = adminLn.Close()
|
||||||
|
_ = agentLn.Close()
|
||||||
|
return terr
|
||||||
|
}
|
||||||
|
// Serve on an already-wrapped listener rather than
|
||||||
|
// ServeTLS: agentSrv.TLSConfig stays nil, which is exactly
|
||||||
|
// the case where net/http sets up HTTP/2 for ALPN itself.
|
||||||
|
agentLn = tls.NewListener(agentLn, tc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
g, gctx := errgroup.WithContext(ctx)
|
g, gctx := errgroup.WithContext(ctx)
|
||||||
g.Go(func() error { return serve(apiSrv, apiLn) })
|
g.Go(func() error { return serve(apiSrv, apiLn) })
|
||||||
g.Go(func() error { return serve(adminSrv, adminLn) })
|
g.Go(func() error { return serve(adminSrv, adminLn) })
|
||||||
|
if agentSrv != nil {
|
||||||
|
g.Go(func() error { return serve(agentSrv, agentLn) })
|
||||||
|
}
|
||||||
g.Go(func() error {
|
g.Go(func() error {
|
||||||
<-gctx.Done()
|
<-gctx.Done()
|
||||||
log.Info("shutting down")
|
log.Info("shutting down")
|
||||||
sctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
sctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
if agentSrv != nil {
|
||||||
|
_ = agentSrv.Shutdown(sctx)
|
||||||
|
}
|
||||||
|
if metricsSrv != nil {
|
||||||
|
_ = metricsSrv.Shutdown(sctx)
|
||||||
|
}
|
||||||
_ = adminSrv.Shutdown(sctx)
|
_ = adminSrv.Shutdown(sctx)
|
||||||
return apiSrv.Shutdown(sctx)
|
return apiSrv.Shutdown(sctx)
|
||||||
})
|
})
|
||||||
@@ -121,12 +295,127 @@ func newServerCmd() *cobra.Command {
|
|||||||
f := cmd.Flags()
|
f := cmd.Flags()
|
||||||
f.StringVar(&configPath, "config", "", "configuration file (optional)")
|
f.StringVar(&configPath, "config", "", "configuration file (optional)")
|
||||||
f.StringVar(&listen, "listen", "", "override the client-facing address")
|
f.StringVar(&listen, "listen", "", "override the client-facing address")
|
||||||
f.StringVar(&adminListen, "admin-listen", "", "override the metrics/pprof address")
|
f.StringVar(&adminListen, "admin-listen", "", "override the dashboard/pprof address")
|
||||||
|
f.StringVar(&metricsListen, "metrics-listen", "", "serve metrics only on this address, safe to expose")
|
||||||
f.StringArrayVar(&endpoints, "endpoint", nil, "static endpoint: id=URL=model[,model] (repeatable)")
|
f.StringArrayVar(&endpoints, "endpoint", nil, "static endpoint: id=URL=model[,model] (repeatable)")
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// startControlPlane builds the control plane and its agent service.
|
||||||
|
//
|
||||||
|
// The service is both the AgentService handler and the reconciler's Dispatcher:
|
||||||
|
// it holds the open streams, so it is the only thing that can deliver a command
|
||||||
|
// to a node.
|
||||||
|
func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live,
|
||||||
|
sink control.EndpointSink, requireIdentity bool, log *slog.Logger,
|
||||||
|
) (*control.Controller, *agentsvc.Service, error) {
|
||||||
|
ccfg := control.DefaultConfig()
|
||||||
|
ccfg.Reconciler = reconciler.Config{
|
||||||
|
Interval: cfg.Control.Interval,
|
||||||
|
BaseBackoff: cfg.Control.BaseBackoff,
|
||||||
|
MaxBackoff: cfg.Control.MaxBackoff,
|
||||||
|
UnreachableAfter: cfg.Control.UnreachableAfter,
|
||||||
|
LostAfter: cfg.Control.LostAfter,
|
||||||
|
MgmtInterface: cfg.Control.MgmtInterface,
|
||||||
|
}
|
||||||
|
if len(cfg.Control.PortRange) == 2 {
|
||||||
|
ccfg.Reconciler.PortRange = [2]int{cfg.Control.PortRange[0], cfg.Control.PortRange[1]}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The service needs the inventory and the status sink, both of which the
|
||||||
|
// controller owns; the controller needs the service as its dispatcher. The
|
||||||
|
// cycle is broken by giving the controller a dispatcher that forwards to
|
||||||
|
// the service once it exists.
|
||||||
|
fwd := &dispatcherRef{}
|
||||||
|
ctl := control.New(ccfg, cat, fwd, sink, log)
|
||||||
|
// The router is both where endpoints go and where activity comes back
|
||||||
|
// from. It only reports usage when it is the thing serving requests, which
|
||||||
|
// is why this is a type assertion rather than a parameter.
|
||||||
|
if u, ok := sink.(control.UsageSource); ok {
|
||||||
|
ctl.SetUsageSource(u)
|
||||||
|
}
|
||||||
|
svc := agentsvc.New(ctl.Inventory(), ctl.Reconciler(), requireIdentity, log)
|
||||||
|
fwd.set(svc)
|
||||||
|
|
||||||
|
ctl.Start(ctx)
|
||||||
|
return ctl, svc, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// metricsAddr renders the metrics listener for the startup line, saying so
|
||||||
|
// plainly when there is none rather than printing an empty string.
|
||||||
|
func metricsAddr(ln net.Listener) string {
|
||||||
|
if ln == nil {
|
||||||
|
return "off (admin listener only)"
|
||||||
|
}
|
||||||
|
return ln.Addr().String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// dispatcherRef lets the controller be built before the service exists.
|
||||||
|
type dispatcherRef struct {
|
||||||
|
mu sync.RWMutex
|
||||||
|
d reconciler.Dispatcher
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *dispatcherRef) set(d reconciler.Dispatcher) {
|
||||||
|
r.mu.Lock()
|
||||||
|
r.d = d
|
||||||
|
r.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *dispatcherRef) get() (reconciler.Dispatcher, error) {
|
||||||
|
r.mu.RLock()
|
||||||
|
defer r.mu.RUnlock()
|
||||||
|
if r.d == nil {
|
||||||
|
return nil, errors.New("the agent service is not ready yet")
|
||||||
|
}
|
||||||
|
return r.d, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveWeights forwards a capability beyond scheduling.
|
||||||
|
//
|
||||||
|
// It has to be spelled out here rather than inherited, because this type
|
||||||
|
// satisfies reconciler.Dispatcher by forwarding each method by hand — so
|
||||||
|
// anything the controller reaches by type assertion stops at this boundary
|
||||||
|
// unless it is forwarded too. A capability added to the service and not to
|
||||||
|
// this type is simply absent at runtime, with nothing at compile time to say
|
||||||
|
// so.
|
||||||
|
func (r *dispatcherRef) RemoveWeights(ctx context.Context, nodeID string, paths []string) error {
|
||||||
|
d, err := r.get()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rm, ok := d.(control.WeightRemover)
|
||||||
|
if !ok {
|
||||||
|
return errors.New("the agent service cannot remove weights")
|
||||||
|
}
|
||||||
|
return rm.RemoveWeights(ctx, nodeID, paths)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *dispatcherRef) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
|
||||||
|
d, err := r.get()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return d.StartShard(ctx, cmd)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *dispatcherRef) StopShard(ctx context.Context, nodeID, shardID string) error {
|
||||||
|
d, err := r.get()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return d.StopShard(ctx, nodeID, shardID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *dispatcherRef) DrainShard(ctx context.Context, nodeID, shardID string) error {
|
||||||
|
d, err := r.get()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return d.DrainShard(ctx, nodeID, shardID)
|
||||||
|
}
|
||||||
|
|
||||||
func serve(s *http.Server, ln net.Listener) error {
|
func serve(s *http.Server, ln net.Listener) error {
|
||||||
if err := s.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
if err := s.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -17,15 +17,39 @@ import (
|
|||||||
func newSimEngineCmd() *cobra.Command {
|
func newSimEngineCmd() *cobra.Command {
|
||||||
cfg := sim.Default()
|
cfg := sim.Default()
|
||||||
var listen string
|
var listen string
|
||||||
|
var rv sim.Rendezvous
|
||||||
|
|
||||||
cmd := &cobra.Command{
|
cmd := &cobra.Command{
|
||||||
Use: "sim-engine",
|
Use: "sim-engine",
|
||||||
Short: "OpenAI-compatible fake inference engine (for testing)",
|
Short: "OpenAI-compatible fake inference engine (for testing)",
|
||||||
Long: "Simulates the engine behaviours that matter to the router:\n" +
|
Long: "Simulates the engine behaviours that matter to the router:\n" +
|
||||||
"continuous batching, KV block accounting, prefix caching, and vLLM-shaped\n" +
|
"continuous batching, KV block accounting, prefix caching, and vLLM-shaped\n" +
|
||||||
"/metrics. Lets the router be developed and tested without GPUs.",
|
"/metrics. Lets the router be developed and tested without GPUs.\n\n" +
|
||||||
|
"With --nnodes above 1 it also simulates a multi-node replica: rank 0\n" +
|
||||||
|
"listens at --dist-init-addr and does not serve until every other rank has\n" +
|
||||||
|
"joined, and only rank 0 answers the API. That is the part of the\n" +
|
||||||
|
"multi-node path stub processes cannot exercise — the sequencing, the\n" +
|
||||||
|
"waiting, and one dead shard taking the replica with it.",
|
||||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
log := logger()
|
log := logger()
|
||||||
|
|
||||||
|
ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
// Before anything binds the API port: a worker must never look
|
||||||
|
// ready, and a leader must not serve until the replica exists.
|
||||||
|
if err := rv.Join(ctx, log); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if rv.Multi() && !rv.Leader() {
|
||||||
|
// A worker contributes GPUs and answers nothing. Health
|
||||||
|
// checking its port is what used to fail replicas that were
|
||||||
|
// coming up perfectly well.
|
||||||
|
log.Info("worker rank running; this rank serves no API", "rank", rv.Rank)
|
||||||
|
<-ctx.Done()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
eng := sim.New(cfg)
|
eng := sim.New(cfg)
|
||||||
defer eng.Close()
|
defer eng.Close()
|
||||||
|
|
||||||
@@ -47,9 +71,6 @@ func newSimEngineCmd() *cobra.Command {
|
|||||||
"prefix_cache_blocks", cfg.PrefixCacheBlocks,
|
"prefix_cache_blocks", cfg.PrefixCacheBlocks,
|
||||||
)
|
)
|
||||||
|
|
||||||
ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM)
|
|
||||||
defer stop()
|
|
||||||
|
|
||||||
errCh := make(chan error, 1)
|
errCh := make(chan error, 1)
|
||||||
go func() { errCh <- srv.Serve(ln) }()
|
go func() { errCh <- srv.Serve(ln) }()
|
||||||
|
|
||||||
@@ -70,6 +91,11 @@ func newSimEngineCmd() *cobra.Command {
|
|||||||
|
|
||||||
f := cmd.Flags()
|
f := cmd.Flags()
|
||||||
f.StringVar(&listen, "listen", ":8000", "address to listen on")
|
f.StringVar(&listen, "listen", ":8000", "address to listen on")
|
||||||
|
f.IntVar(&rv.NNodes, "nnodes", 1, "ranks in this replica; above 1 simulates a multi-node launch")
|
||||||
|
f.IntVar(&rv.Rank, "node-rank", 0, "this rank; 0 is the leader and the only one that serves")
|
||||||
|
f.StringVar(&rv.Addr, "dist-init-addr", "", "the leader's rendezvous address")
|
||||||
|
f.DurationVar(&rv.Timeout, "rendezvous-timeout", 5*time.Minute,
|
||||||
|
"how long to wait for the replica to assemble before failing")
|
||||||
f.StringVar(&cfg.Model, "model", cfg.Model, "model name to serve")
|
f.StringVar(&cfg.Model, "model", cfg.Model, "model name to serve")
|
||||||
f.IntVar(&cfg.MaxNumSeqs, "max-num-seqs", cfg.MaxNumSeqs, "concurrent sequences (continuous batch capacity)")
|
f.IntVar(&cfg.MaxNumSeqs, "max-num-seqs", cfg.MaxNumSeqs, "concurrent sequences (continuous batch capacity)")
|
||||||
f.IntVar(&cfg.BlockSize, "block-size", cfg.BlockSize, "tokens per KV cache block")
|
f.IntVar(&cfg.BlockSize, "block-size", cfg.BlockSize, "tokens per KV cache block")
|
||||||
|
|||||||
@@ -0,0 +1,163 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||||
|
)
|
||||||
|
|
||||||
|
// newStatusCmd prints the whole fleet in one screen.
|
||||||
|
//
|
||||||
|
// `ctl nodes` is the right tool for four nodes and the wrong one for two
|
||||||
|
// hundred. The questions that matter at that size are aggregates — how full am
|
||||||
|
// I, what is broken, what is holding cards nobody is using — and reading them
|
||||||
|
// off a long table is a chore, which means it gets skipped, which means nobody
|
||||||
|
// notices until something fails.
|
||||||
|
func newStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "status",
|
||||||
|
Short: "The whole fleet in one answer: capacity, health, and what is idle",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
var s adminapi.Summary
|
||||||
|
if err := client().get(cmd.Context(), "/api/v1/summary", &s); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
return emitJSON(s)
|
||||||
|
}
|
||||||
|
printSummary(cmd, s)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func printSummary(cmd *cobra.Command, s adminapi.Summary) {
|
||||||
|
out := cmd.OutOrStdout()
|
||||||
|
|
||||||
|
fmt.Fprintf(out, "NODES %d ready", s.Nodes.Ready)
|
||||||
|
for _, x := range []struct {
|
||||||
|
n int
|
||||||
|
label string
|
||||||
|
}{
|
||||||
|
{s.Nodes.Cordoned, "cordoned"},
|
||||||
|
{s.Nodes.Draining, "draining"},
|
||||||
|
{s.Nodes.Unreachable, "unreachable"},
|
||||||
|
} {
|
||||||
|
if x.n > 0 {
|
||||||
|
fmt.Fprintf(out, ", %d %s", x.n, x.label)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, " (of %d)\n", s.Nodes.Total)
|
||||||
|
|
||||||
|
if len(s.GPUs) > 0 {
|
||||||
|
fmt.Fprintln(out, "\nGPUS")
|
||||||
|
w := tab()
|
||||||
|
fmt.Fprintln(w, " CLASS\tIN USE\tFREE\tUNHEALTHY\tUSAGE")
|
||||||
|
for _, g := range s.GPUs {
|
||||||
|
fmt.Fprintf(w, " %s\t%d/%d\t%d\t%s\t%s\n",
|
||||||
|
g.Class, g.Used, g.Total, g.Free(), unhealthyCell(g.Unhealthy), bar(g.Used, g.Total))
|
||||||
|
}
|
||||||
|
_ = w.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(s.Fabrics) > 0 {
|
||||||
|
fmt.Fprintln(out, "\nFABRICS")
|
||||||
|
w := tab()
|
||||||
|
// The free-block column is the one that decides whether a multi-node
|
||||||
|
// replica can land: free cards on the wrong fabric are not capacity for
|
||||||
|
// a model that spans nodes, because a replica may never cross one.
|
||||||
|
fmt.Fprintln(w, " FABRIC\tNODES\tFULLY FREE\tWEIGHTS")
|
||||||
|
for _, f := range s.Fabrics {
|
||||||
|
fmt.Fprintf(w, " %s\t%d\t%d\t%s\n",
|
||||||
|
f.Fabric, f.Nodes, f.LargestFreeBlock, weightsCell(f))
|
||||||
|
}
|
||||||
|
_ = w.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintf(out, "\nREPLICAS %d ready", s.Replicas.Ready)
|
||||||
|
for _, x := range []struct {
|
||||||
|
n int
|
||||||
|
label string
|
||||||
|
}{
|
||||||
|
{s.Replicas.Starting, "starting"},
|
||||||
|
{s.Replicas.Draining, "draining"},
|
||||||
|
{s.Replicas.Failed, "failed"},
|
||||||
|
} {
|
||||||
|
if x.n > 0 {
|
||||||
|
fmt.Fprintf(out, ", %d %s", x.n, x.label)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "\nMODELS %d of %d have the replicas they asked for\n",
|
||||||
|
s.Models.Satisfied, s.Models.Total)
|
||||||
|
|
||||||
|
if len(s.Unplaceable) > 0 {
|
||||||
|
fmt.Fprintln(out, "\nNOT PLACED")
|
||||||
|
for _, m := range s.Unplaceable {
|
||||||
|
fmt.Fprintf(out, " %s\n", m)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "\n Why, per model: llmgw ctl explain-placement <model>")
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(s.IdleReplicas) > 0 {
|
||||||
|
fmt.Fprintln(out, "\nIDLE")
|
||||||
|
w := tab()
|
||||||
|
fmt.Fprintln(w, " REPLICA\tMODEL\tGPUS\tUNUSED FOR")
|
||||||
|
for _, r := range s.IdleReplicas {
|
||||||
|
fmt.Fprintf(w, " %s\t%s\t%d\t%s\n", r.ID, r.Model, r.GPUs,
|
||||||
|
(time.Duration(r.IdleSec) * time.Second).Round(time.Minute))
|
||||||
|
}
|
||||||
|
_ = w.Flush()
|
||||||
|
fmt.Fprintln(out, "\n These hold hardware nothing is asking for.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// weightsCell says what a fabric holds, and stays quiet when nobody asked.
|
||||||
|
//
|
||||||
|
// Fabrics are not connected to one another, so weights in one are not weights
|
||||||
|
// in another: a model seeded into a single fabric leaves the rest of the
|
||||||
|
// cluster unable to serve it without first moving tens of gigabytes over the
|
||||||
|
// management network. This is the column that shows that before someone
|
||||||
|
// discovers it during a scale-up.
|
||||||
|
func weightsCell(f adminapi.FabricUse) string {
|
||||||
|
if f.NodesWithWeights == 0 {
|
||||||
|
// Nobody on this fabric was asked to look, which is the default and is
|
||||||
|
// not the same as an empty fabric.
|
||||||
|
return "-"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%d model(s) on %d/%d nodes", f.Models, f.NodesWithWeights, f.Nodes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// bar draws usage so a column of classes can be read at a glance rather than
|
||||||
|
// compared digit by digit.
|
||||||
|
func bar(used, total int) string {
|
||||||
|
const width = 20
|
||||||
|
if total <= 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
filled := used * width / total
|
||||||
|
if filled > width {
|
||||||
|
filled = width
|
||||||
|
}
|
||||||
|
b := make([]rune, width)
|
||||||
|
for i := range b {
|
||||||
|
if i < filled {
|
||||||
|
b[i] = '#'
|
||||||
|
} else {
|
||||||
|
b[i] = '.'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s %3d%%", string(b), used*100/total)
|
||||||
|
}
|
||||||
|
|
||||||
|
// unhealthyCell keeps a zero quiet. A column of noughts trains the eye to skip
|
||||||
|
// the column, which is the opposite of what it is for.
|
||||||
|
func unhealthyCell(n int) string {
|
||||||
|
if n == 0 {
|
||||||
|
return "-"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%d", n)
|
||||||
|
}
|
||||||
@@ -0,0 +1,180 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBarStaysWithinItsWidth(t *testing.T) {
|
||||||
|
// It is drawn from counts that can disagree briefly — a GPU freed between
|
||||||
|
// two reads — and a bar that overruns its column wrecks the table it is in.
|
||||||
|
for _, tc := range [][2]int{{0, 0}, {0, 8}, {8, 8}, {9, 8}, {100, 8}} {
|
||||||
|
got := bar(tc[0], tc[1])
|
||||||
|
if tc[1] == 0 {
|
||||||
|
if got != "" {
|
||||||
|
t.Errorf("bar(%d,%d) = %q, want empty for an empty class", tc[0], tc[1], got)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if n := strings.Count(got, "#") + strings.Count(got, "."); n != 20 {
|
||||||
|
t.Errorf("bar(%d,%d) drew %d cells: %q", tc[0], tc[1], n, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIdleColumnDistinguishesUnknownFromIdle(t *testing.T) {
|
||||||
|
// Every replica looks unreported before its first request and after a
|
||||||
|
// router restart. Showing that as a very large duration would make an
|
||||||
|
// untouched fleet read as an abandoned one.
|
||||||
|
if got := idleColumn(-1); got != "-" {
|
||||||
|
t.Errorf("idleColumn(-1) = %q, want a dash", got)
|
||||||
|
}
|
||||||
|
if got := idleColumn(90); got != "1m30s" {
|
||||||
|
t.Errorf("idleColumn(90) = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestModelStatusSeparatesWaitingFromBroken(t *testing.T) {
|
||||||
|
// These used to differ by a parenthetical — "0/1 ready (1 total)" against
|
||||||
|
// "0/1 ready" — and they need opposite responses: one means wait, the other
|
||||||
|
// means look at explain-placement. On forty models that difference is
|
||||||
|
// invisible.
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
view adminapi.ModelView
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"serving what it asked for", adminapi.ModelView{Min: 2, Ready: 2, Total: 2}, "ready"},
|
||||||
|
{"a replica is coming up", adminapi.ModelView{Min: 1, Ready: 0, Total: 1}, "starting"},
|
||||||
|
{"nothing could be placed", adminapi.ModelView{Min: 1, Ready: 0, Total: 0}, "not placed"},
|
||||||
|
{"deliberately not running", adminapi.ModelView{Min: 0, Ready: 0, Total: 0}, "stopped"},
|
||||||
|
{"short but rebuilding", adminapi.ModelView{Min: 2, Ready: 1, Total: 2}, "starting"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if got := modelStatus(tc.view); got != tc.want {
|
||||||
|
t.Errorf("modelStatus(%+v) = %q, want %q", tc.view, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHardwarePoolsNeverMergeAcrossFabrics(t *testing.T) {
|
||||||
|
// A replica may never span two fabrics, so eight free cards split across
|
||||||
|
// two of them are not eight cards for a model that needs eight. Merging
|
||||||
|
// them would plan something that starts, passes health checks and hangs on
|
||||||
|
// its first all-reduce.
|
||||||
|
nodes := []adminapi.NodeView{
|
||||||
|
{ID: "a", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-a"}, GPUFree: 4, GPUTotal: 4},
|
||||||
|
{ID: "b", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-a"}, GPUFree: 4, GPUTotal: 4},
|
||||||
|
{ID: "c", State: "ready", Classes: []string{"h100-80g"}, Fabrics: []string{"ib-b"}, GPUFree: 4, GPUTotal: 4},
|
||||||
|
}
|
||||||
|
hw := hardwareFrom(nodes)
|
||||||
|
if len(hw) != 2 {
|
||||||
|
t.Fatalf("expected one pool per fabric, got %d: %+v", len(hw), hw)
|
||||||
|
}
|
||||||
|
for _, p := range hw {
|
||||||
|
switch p.Fabric {
|
||||||
|
case "ib-a":
|
||||||
|
if p.Nodes != 2 {
|
||||||
|
t.Errorf("ib-a has %d nodes, want 2", p.Nodes)
|
||||||
|
}
|
||||||
|
case "ib-b":
|
||||||
|
if p.Nodes != 1 {
|
||||||
|
t.Errorf("ib-b has %d nodes, want 1", p.Nodes)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
t.Errorf("unexpected fabric %q", p.Fabric)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnreadyNodesAreNotCountedAsCapacity(t *testing.T) {
|
||||||
|
// Planning against a node that is draining or unreachable produces a fit
|
||||||
|
// that cannot be placed, which is worse than saying nothing fits.
|
||||||
|
nodes := []adminapi.NodeView{
|
||||||
|
{ID: "a", State: "ready", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8},
|
||||||
|
{ID: "b", State: "draining", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8},
|
||||||
|
{ID: "c", State: "unreachable", Classes: []string{"h100-80g"}, GPUFree: 8, GPUTotal: 8},
|
||||||
|
}
|
||||||
|
hw := hardwareFrom(nodes)
|
||||||
|
if len(hw) != 1 || hw[0].Nodes != 1 {
|
||||||
|
t.Fatalf("expected only the ready node, got %+v", hw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCardMemoryComesFromTheClassLabel(t *testing.T) {
|
||||||
|
// The label carries it by construction — "h100-80g" is eighty gigabytes and
|
||||||
|
// says so — which is why the planner needs no table of card models.
|
||||||
|
for label, want := range map[string]int{
|
||||||
|
"h100-80g": 80, "a100-40g": 40, "l40s-48g": 48, "rtx4090-24g": 24,
|
||||||
|
"nonsense": 0, "": 0, "h100-": 0,
|
||||||
|
} {
|
||||||
|
if got := memFromClass(label); got != want {
|
||||||
|
t.Errorf("memFromClass(%q) = %d, want %d", label, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestACPUOnlyNodeBecomesAMemoryPool(t *testing.T) {
|
||||||
|
// How this cluster began: a node with no cards is not an empty pool, it is
|
||||||
|
// a pool measured in RAM.
|
||||||
|
hw := hardwareFrom([]adminapi.NodeView{
|
||||||
|
{ID: "cpu1", State: "ready", MemoryGB: 31},
|
||||||
|
})
|
||||||
|
if len(hw) != 1 {
|
||||||
|
t.Fatalf("the CPU node produced %d pools", len(hw))
|
||||||
|
}
|
||||||
|
if hw[0].Class != "" || hw[0].HostMemGB != 31 {
|
||||||
|
t.Errorf("pool = %+v, want a classless pool with 31 GB", hw[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAGPUNodeWithNoFreeCardsIsNotAPool(t *testing.T) {
|
||||||
|
// Fully allocated hardware is not capacity, and offering it would plan a
|
||||||
|
// replica that cannot be placed.
|
||||||
|
hw := hardwareFrom([]adminapi.NodeView{
|
||||||
|
{ID: "full", State: "ready", Classes: []string{"h100-80g"}, GPUFree: 0, GPUTotal: 8},
|
||||||
|
})
|
||||||
|
for _, p := range hw {
|
||||||
|
if p.Class != "" {
|
||||||
|
t.Errorf("a node with no free cards became a pool: %+v", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheEntryNamesAnEngineThatCanServeThePrecision(t *testing.T) {
|
||||||
|
// The defect this prevents, found by reading the tool's own output: it
|
||||||
|
// chose q8_0 — a GGUF quantisation — and emitted a vLLM profile with no
|
||||||
|
// quantisation at all. Following that loads full-precision weights, which
|
||||||
|
// the same plan had just shown do not fit on one card. The tool would have
|
||||||
|
// caused the out-of-memory it exists to prevent.
|
||||||
|
m := modelplan.Model{Name: "Qwen2.5-Coder-32B", Params: 32_800_000_000, Layers: 64, KVHeads: 8, HeadDim: 128}
|
||||||
|
|
||||||
|
gguf := modelplan.Fit{Precision: modelplan.Q8GGUF, Class: "h100-80g", TP: 1, Nodes: 1}
|
||||||
|
entry := catalogEntry(m, gguf, "Qwen/Qwen2.5-Coder-32B-Instruct", 32768)
|
||||||
|
if !strings.Contains(entry, "engine: llamacpp") {
|
||||||
|
t.Errorf("a GGUF precision produced a non-llamacpp profile:\n%s", entry)
|
||||||
|
}
|
||||||
|
if strings.Contains(entry, "source: Qwen/") {
|
||||||
|
t.Errorf("a GGUF entry points at a repository rather than a file:\n%s", entry)
|
||||||
|
}
|
||||||
|
if !strings.Contains(entry, ".gguf") {
|
||||||
|
t.Errorf("a GGUF entry has no file path:\n%s", entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
fp8 := modelplan.Fit{Precision: modelplan.FP8, Class: "h100-80g", TP: 2, Nodes: 1}
|
||||||
|
entry = catalogEntry(m, fp8, "Qwen/Qwen2.5-Coder-32B-Instruct", 32768)
|
||||||
|
if !strings.Contains(entry, "engine: vllm") {
|
||||||
|
t.Errorf("fp8 did not produce a vLLM profile:\n%s", entry)
|
||||||
|
}
|
||||||
|
// And the quantisation has to reach the engine, or vLLM loads bf16.
|
||||||
|
if !strings.Contains(entry, "--quantization") || !strings.Contains(entry, "fp8") {
|
||||||
|
t.Errorf("the fp8 quantisation is not passed to the engine:\n%s", entry)
|
||||||
|
}
|
||||||
|
if !strings.Contains(entry, "source: Qwen/") {
|
||||||
|
t.Errorf("a vLLM entry should load from the repository:\n%s", entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
Loaded 100 of 269 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user