Compare commits
No files matched your search
@@ -93,6 +93,24 @@ jobs:
|
|||||||
./dist/llmgw-${version}-linux-amd64 --version
|
./dist/llmgw-${version}-linux-amd64 --version
|
||||||
cat dist/checksums.txt
|
cat dist/checksums.txt
|
||||||
|
|
||||||
|
# Signed when the repository has a signing key, so a machine that trusts
|
||||||
|
# it installs nothing the server alone could have put there. Without the
|
||||||
|
# secret nothing is signed and releases are checked on their checksums,
|
||||||
|
# as before; a build that trusts a key refuses those, so the secret has
|
||||||
|
# to be in place before trusted.pub is filled in.
|
||||||
|
- name: Sign the checksums
|
||||||
|
env:
|
||||||
|
RELEASE_SIGNING_KEY: ${{ secrets.RELEASE_SIGNING_KEY }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
version="${GITHUB_REF_NAME:-$(git describe --tags --always)}"
|
||||||
|
if [ -z "${RELEASE_SIGNING_KEY:-}" ]; then
|
||||||
|
echo "no RELEASE_SIGNING_KEY secret; releases are not signed"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
./dist/llmgw-${version}-linux-amd64 release sign dist/checksums.txt
|
||||||
|
echo "signed: dist/checksums.txt.sig"
|
||||||
|
|
||||||
- name: Attach to the release
|
- name: Attach to the release
|
||||||
env:
|
env:
|
||||||
TOKEN: ${{ secrets.GITEA_TOKEN }}
|
TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
|||||||
+545
@@ -20,6 +20,551 @@ is only released once its **exit criterion** is met, not when its code lands.
|
|||||||
Until v1.0.0 the minor version is the breaking one: configuration formats and Go
|
Until v1.0.0 the minor version is the breaking one: configuration formats and Go
|
||||||
APIs change between minors without a deprecation period.
|
APIs change between minors without a deprecation period.
|
||||||
|
|
||||||
|
## v0.10.6 — 2026-10-01
|
||||||
|
|
||||||
|
Published first as release candidates v0.10.6-rc.1 to rc.8; rc.5 failed its
|
||||||
|
format check and has no binaries.
|
||||||
|
|
||||||
|
Tried on the PoC cluster before release: LoRA adapters asked for by name
|
||||||
|
through the gateway, two on one model, each answering as trained and the model
|
||||||
|
itself unchanged; a saturated pool spilling to the next (8 of 12 concurrent
|
||||||
|
requests on the preferred replica, 4 on the other); and a bronze request
|
||||||
|
displaced from a full queue by a gold one, with 503 and Retry-After 5. Not tried
|
||||||
|
live: a model served from an external provider, which needs a provider key.
|
||||||
|
|
||||||
|
### Capacity from the admin pages
|
||||||
|
|
||||||
|
What v0.10.5 made settable from the command line is now on the pages too:
|
||||||
|
|
||||||
|
- **Quotas** has a Capacity section: every scope's guaranteed and most GPUs,
|
||||||
|
cores and memory beside what it holds, with a form to set, change or remove
|
||||||
|
one. It goes through the same checks as `llmgw ctl quota`.
|
||||||
|
- **A model's card** on the mirror page sets its class, its owner (when there
|
||||||
|
are quotas to count against), and the memory and cores a replica reserves,
|
||||||
|
beside its replica counts. Only what changed is applied, and each change is
|
||||||
|
said.
|
||||||
|
- **Room counts what a class may take.** The card, `llmgw ctl model target` and
|
||||||
|
the headroom API count the replicas a model could get by stopping those its
|
||||||
|
class outranks, separately from free room, and name whose they would be. A
|
||||||
|
critical model's card no longer refuses a replica that preemption would make
|
||||||
|
room for.
|
||||||
|
|
||||||
|
### LoRA adapters
|
||||||
|
|
||||||
|
A llama.cpp model can carry `adapters`, each a name and a GGUF LoRA file. The
|
||||||
|
nodes fetch them with the model's weights and llama.cpp loads them all at start,
|
||||||
|
applied to nothing (`--lora-scaled FILE:0`); a request that names an
|
||||||
|
adapter goes to the model's replicas with `"lora": [{"id": i, "scale": s}]`,
|
||||||
|
so one replica serves the model and all its adapters. Adapter names share the
|
||||||
|
namespace of models, are listed in `/v1/models`, and follow key allowlists on
|
||||||
|
their own — a key allowed an adapter does not reach the model under it. Adapter
|
||||||
|
files are kept from weight cleanup like the model's own. Not yet for vLLM or
|
||||||
|
SGLang profiles, whose adapters are loaded differently.
|
||||||
|
|
||||||
|
### Finding and attaching adapters from the mirror page
|
||||||
|
|
||||||
|
- **Explore** has a *LoRA adapters* view: adapters for any model, or — from a
|
||||||
|
held model's card, *find adapters on Hugging Face* — those made for the model
|
||||||
|
its repository was made from, since a GGUF repository is a quantisation and
|
||||||
|
adapters are made for the original.
|
||||||
|
- **Opening a repository** reads the first bytes of each GGUF file and marks it
|
||||||
|
a *LoRA adapter* or a *model*. Repositories tagged "lora" hold a model with
|
||||||
|
the adapter merged in about as often as an adapter, and only the file says.
|
||||||
|
- **Held adapter files** are marked in the list, and a model's card lists its
|
||||||
|
adapters, offers the held adapter files of its architecture to attach under a
|
||||||
|
name and scale, and removes them. A file that is not an adapter, or is for
|
||||||
|
another architecture, is refused before it reaches the catalog.
|
||||||
|
- **`llmgw ctl model adapters MODEL [--add NAME=URL[@SCALE]] [--remove NAME]`**
|
||||||
|
lists and changes them from the command line, through
|
||||||
|
`POST /api/v1/models/{name}/adapters`, with the same checks.
|
||||||
|
|
||||||
|
### Models served outside the cluster
|
||||||
|
|
||||||
|
A catalog model can be an OpenAI-compatible API elsewhere: `external: {url,
|
||||||
|
model, api_key_file}` and no profiles. Nothing is placed for it; the router
|
||||||
|
sends its requests to the provider, rewritten to the provider's name for the
|
||||||
|
model and with its key, read from the file on every publish so a rotated key is
|
||||||
|
picked up. Named in another model's `fallbacks`, it is used only when the
|
||||||
|
cluster has no capacity left. It is never scraped for metrics, and never reads
|
||||||
|
as stale. Prices, budgets, limits, usage and key allowlists apply as to any
|
||||||
|
model — and since its prompts leave the cluster, the allowlist is how to decide
|
||||||
|
who may send them (docs/install.md). `ctl models` and the cluster page mark it
|
||||||
|
external.
|
||||||
|
|
||||||
|
### Pool spillover
|
||||||
|
|
||||||
|
A model served from pools of different priority — H100 before A100, say — was
|
||||||
|
routed to the preferred pool however full it was. Now a pool whose replicas are
|
||||||
|
saturated, by the same measure admission uses (queue past its healthy depth, or
|
||||||
|
KV cache past its ceiling), passes requests to the next pool that is not. When
|
||||||
|
every pool is saturated the preferred one is kept: a pool that is just as full
|
||||||
|
gains nothing.
|
||||||
|
|
||||||
|
### Displacement: a higher band is not refused while a lower one waits
|
||||||
|
|
||||||
|
Each band's queue was bounded, and nothing bounded them together, so a full
|
||||||
|
gateway turned an interactive request away while batch requests sat queued
|
||||||
|
behind it. `admission.queue_capacity_total` now bounds everything queued —
|
||||||
|
by default half of every band's capacity together. Reached, a request is not
|
||||||
|
refused while a lower band has any queued: the newest request of the lowest
|
||||||
|
band is turned away instead, with a 503 and `Retry-After: 5`, so it backs off
|
||||||
|
and returns. The newest, because the oldest have waited longest and are
|
||||||
|
nearest their TTL. Counted as `displaced` in the admission rejection metric.
|
||||||
|
|
||||||
|
### Releases can be signed
|
||||||
|
|
||||||
|
`llmgw release keygen` makes an ed25519 key pair. With the private key in the
|
||||||
|
CI secret `RELEASE_SIGNING_KEY`, releases carry `checksums.txt.sig`; with the
|
||||||
|
public key in `internal/release/trusted.pub`, every binary built after refuses
|
||||||
|
a release not signed with it, wherever it installs one. Off until then, and
|
||||||
|
docs/install.md says in which order to turn it on.
|
||||||
|
|
||||||
|
### Restarts borrow a lower class's room
|
||||||
|
|
||||||
|
A restart places the replacement beside the replica it replaces, and where
|
||||||
|
there is no free room it now makes some the way any placement does: by
|
||||||
|
stopping a replica its class outranks, which gets its room back once the old
|
||||||
|
replica has gone. The in-place restart — the model unserved while its
|
||||||
|
replacement loads — is left for when nothing of a lower class is running. On a
|
||||||
|
cluster where every model is standard, that is every full one: mark the models
|
||||||
|
that can wait `opportunistic` (`llmgw ctl model class`, or the model's card).
|
||||||
|
|
||||||
|
## v0.10.5 — 2026-09-30
|
||||||
|
|
||||||
|
Published first as release candidates v0.10.5-rc.1 to rc.14; rc.5 failed its
|
||||||
|
format check and has no binaries.
|
||||||
|
|
||||||
|
After upgrading:
|
||||||
|
|
||||||
|
- **Declare what CPU models need.** A CPU model added from now on must say its
|
||||||
|
memory; one stored before this still loads and is flagged in `ctl models`
|
||||||
|
until `llmgw ctl model resources MODEL --memory GB --cores N` is run. Cores
|
||||||
|
are then reserved per replica and become the engine's thread count, for
|
||||||
|
replicas started after the change: `llmgw ctl restart --model MODEL`.
|
||||||
|
- **Upgrade the mirror on its own host**, now with one command:
|
||||||
|
`sudo llmgw upgrade v0.10.5 --service llmgw-mirror` (by hand once, for a
|
||||||
|
mirror older than this). `llmgw ctl upgrade` says so at the end.
|
||||||
|
|
||||||
|
### CPU cores are reserved, and memory has to be declared
|
||||||
|
|
||||||
|
The first step of resource allocation for larger clusters: what a replica
|
||||||
|
holds of a node is reserved, not assumed.
|
||||||
|
|
||||||
|
- **Cores.** A profile's `cpu_cores` is reserved out of its node's CPUs, as
|
||||||
|
memory is, and is what its engine computes on (llama.cpp's `--threads`). A
|
||||||
|
node holds as many CPU replicas as it has cores for, and two on one node no
|
||||||
|
longer spin against each other. Agents report their CPU count; a node that
|
||||||
|
does not, and a profile that declares no cores, constrain nothing.
|
||||||
|
- **Memory is required of new CPU models.** A model with no GPU is all host
|
||||||
|
memory, and one declaring none looked to placement as if it fitted
|
||||||
|
everywhere, any number of times. Adding one without `min_memory_gb` is now
|
||||||
|
refused. Models stored before this still load, and say what is missing.
|
||||||
|
- **Declared against measured.** Each model shows what one replica reserves
|
||||||
|
and the most one of its replicas uses — in `ctl models` (RESERVES) and on the
|
||||||
|
cluster page — with a warning when nothing is declared or when it uses more
|
||||||
|
than a tenth over what it reserves.
|
||||||
|
- **Setting them.** `llmgw ctl model resources MODEL --memory GB --cores N`
|
||||||
|
(and `POST /api/v1/models/{name}/resources`) for a model added at runtime.
|
||||||
|
`ctl model serve` and the mirror's serve form fill in cores — half the
|
||||||
|
smallest node the model fits on — beside the memory they already estimate.
|
||||||
|
Running replicas keep their old figures until restarted.
|
||||||
|
|
||||||
|
The commands are `llmgw ctl model resources` and `llmgw ctl model class`; in
|
||||||
|
rc.7 the first was briefly `llmgw ctl resources`.
|
||||||
|
|
||||||
|
### Capacity classes, and preemption
|
||||||
|
|
||||||
|
A model has a class — `critical`, `standard` or `opportunistic` — saying what
|
||||||
|
its capacity is worth against another's when there is not enough for both.
|
||||||
|
Unset, an on-demand model is opportunistic and every other model standard,
|
||||||
|
which is how the catalog behaved before. `llmgw ctl model class MODEL CLASS`
|
||||||
|
sets it.
|
||||||
|
|
||||||
|
When a model cannot be placed, replicas are stopped to make room for it, by
|
||||||
|
these rules:
|
||||||
|
|
||||||
|
- A lower class gives way to a higher one.
|
||||||
|
- Within a class, only replicas above a model's minimum — room it grew into
|
||||||
|
under load — and only for a model still below its own minimum.
|
||||||
|
- A critical model's minimum is never taken, nor a replica that has only just
|
||||||
|
loaded, nor one spanning several nodes.
|
||||||
|
- Idle on-demand replicas go first, then what lower classes grew into, then
|
||||||
|
the same within the class, then lower classes' minimums; the longest unused
|
||||||
|
first within each, and of equally idle ones the smallest.
|
||||||
|
- The set is tried on a copy of the inventory first, at most three replicas,
|
||||||
|
and nothing is stopped unless it makes the room. Of a set that does, each
|
||||||
|
replica whose stopping turns out not to be needed is left running, so an
|
||||||
|
idle replica tried first on the wrong node is not stopped with the one that
|
||||||
|
made the room.
|
||||||
|
|
||||||
|
A replica stopped this way is drained and given 30 seconds to finish what it
|
||||||
|
is serving. Its model is held back for up to five minutes, or until the model
|
||||||
|
it gave way to has its replicas, so it does not take the room straight back.
|
||||||
|
`ctl models` shows each model's class and "gave room to X" while held; the
|
||||||
|
cluster page shows the class and how many replicas were lost; the metric is
|
||||||
|
`llmgw_model_preempted_total{model,class}`, and every preemption is logged.
|
||||||
|
|
||||||
|
### Capacity quotas for the tenancy tree
|
||||||
|
|
||||||
|
An organization, team or project can be guaranteed GPUs, CPU cores and
|
||||||
|
memory, and limited to at most some amount of each. What it holds is what the
|
||||||
|
replicas of the models it owns hold — and those of the teams and projects
|
||||||
|
beneath it. `llmgw ctl model owner MODEL team:Acme/ml` says whose a model is;
|
||||||
|
one with no owner is shared and counts against nobody.
|
||||||
|
|
||||||
|
- **A guarantee is kept.** A replica within its owner's guarantee is never
|
||||||
|
taken for another scope's model, whatever the classes: a guarantee a higher
|
||||||
|
class elsewhere could take would not be one.
|
||||||
|
- **Beyond it is borrowing.** Free room is used; when a scope within its own
|
||||||
|
guarantee needs room, what others hold beyond theirs is taken back first,
|
||||||
|
whatever their class — except a critical model's minimum.
|
||||||
|
- **A limit is a ceiling.** A model whose owner would go past a limit anywhere
|
||||||
|
up the tree gets no further replica, and says which limit; headroom and the
|
||||||
|
mirror card stop there too.
|
||||||
|
|
||||||
|
`llmgw ctl quota set team:Acme/ml --gpus 8 --limit-gpus 16` (also `--cores`,
|
||||||
|
`--memory` and their limits), `llmgw ctl quota list` with what each scope holds,
|
||||||
|
`llmgw ctl quota clear`. Scopes are named by path or ID; a name that could mean
|
||||||
|
two is refused. Changes take effect within half a minute. Metrics:
|
||||||
|
`llmgw_capacity_holds`, `llmgw_capacity_guarantee` and `llmgw_capacity_limit`
|
||||||
|
by scope and resource; alert: CapacityAtLimit.
|
||||||
|
|
||||||
|
### Sized by what a replica was measured to serve
|
||||||
|
|
||||||
|
What `llmgw ctl bench MODEL --save` measured — the most requests at once one
|
||||||
|
replica served while meeting its SLO, and its throughput there — now drives
|
||||||
|
decisions rather than only being printed:
|
||||||
|
|
||||||
|
- **Scaling up** when requests per replica pass what was measured, not only
|
||||||
|
when the engines' queues fill: an engine takes requests into its slots long
|
||||||
|
after latency has gone.
|
||||||
|
- **Scaling down** only when the replicas left would stay under 70% of what
|
||||||
|
they were measured to hold, so a replica taken back does not put the rest
|
||||||
|
straight over.
|
||||||
|
- **Choosing a profile.** Among profiles of equal priority, the one measured
|
||||||
|
to serve the most per GPU — or per core — is tried first. Priority set by an
|
||||||
|
operator still wins.
|
||||||
|
- **Sizing.** `llmgw ctl model target MODEL --tokens-per-sec N` says how many
|
||||||
|
replicas that takes, how many run, and whether the rest fit, with the reason
|
||||||
|
when they do not; `--apply` sets it if they do. Headroom reports what one
|
||||||
|
more replica would serve.
|
||||||
|
|
||||||
|
A model never measured behaves exactly as before. Measurements are read from
|
||||||
|
the store every half minute.
|
||||||
|
|
||||||
|
### Upgrading a machine the control plane does not reach
|
||||||
|
|
||||||
|
`sudo llmgw upgrade VERSION --service UNIT` upgrades the machine it runs on —
|
||||||
|
the mirror's host above all, whose service runs unprivileged and cannot replace
|
||||||
|
its own binary as an agent does. The release's checksums are checked, the old
|
||||||
|
binary kept, the unit restarted and watched until it stays up.
|
||||||
|
|
||||||
|
### Nodes report their own heat and power
|
||||||
|
|
||||||
|
The first step of facility-aware placement (docs/facility-aware-placement.md),
|
||||||
|
observe-only: nothing is decided on these yet. Every agent reads, at most every
|
||||||
|
ten seconds, and sends with its heartbeat:
|
||||||
|
|
||||||
|
- **GPUs** from nvidia-smi: temperature, HBM temperature where the driver has
|
||||||
|
it, power draw and the enforced limit, and what is holding the clocks down —
|
||||||
|
power cap, thermal slowdown, power brake. A GPU's temperature was read once,
|
||||||
|
at registration; it is now current.
|
||||||
|
- **CPUs** from hwmon, which is what lm_sensors reads, with nothing to install:
|
||||||
|
the package temperature, and the board's inlet air sensor where it has one.
|
||||||
|
- **CPU power** from RAPL's energy counters.
|
||||||
|
|
||||||
|
A sensor a machine does not have is not reported, never shown as zero. New
|
||||||
|
metrics: `llmgw_gpu_power_watts`, `llmgw_gpu_power_limit_watts`,
|
||||||
|
`llmgw_gpu_memory_temperature_celsius`, `llmgw_gpu_throttled{reason}`,
|
||||||
|
`llmgw_node_cpu_temperature_celsius`, `llmgw_node_cpu_power_watts`,
|
||||||
|
`llmgw_node_inlet_temperature_celsius`. Each node's page shows them, with a GPU
|
||||||
|
table of temperature, power against its limit, and what is throttling it.
|
||||||
|
|
||||||
|
### The mirror page in tabs
|
||||||
|
|
||||||
|
/admin/mirror was one long page: what the mirror holds, the Hugging Face
|
||||||
|
explorer and the fetch form, one under another, with every visit waiting on
|
||||||
|
Hugging Face for a list it might not show. It is now three tabs — Held (with
|
||||||
|
how many are held, downloading and failed), Explore Hugging Face, and Fetch by
|
||||||
|
URL — each with its own address. Only the tab shown does its work, and only
|
||||||
|
Held reloads while something downloads. A download started from the explorer
|
||||||
|
links to Held, where its progress is; a fetch the mirror refused comes back on
|
||||||
|
the form it was sent from.
|
||||||
|
|
||||||
|
### A held model opens to where it runs, and how many
|
||||||
|
|
||||||
|
Clicking a model under Held opens a card beneath it: each replica with its
|
||||||
|
node, state, age and the CPU and memory its engine uses, and how many more
|
||||||
|
replicas the cluster has room for now, on which nodes, and why not more. The
|
||||||
|
replicas it always runs and the most it grows to under load are changed there.
|
||||||
|
Room is found by trying the scheduler itself on a copy of the inventory — the
|
||||||
|
same decisions the next passes would make, not an estimate beside them — and a
|
||||||
|
number of always-running replicas it cannot place is refused with the reason,
|
||||||
|
rather than accepted and left short. A ceiling above the room is taken, and
|
||||||
|
said. The same answer is at `GET /api/v1/models/{name}/headroom` on the admin
|
||||||
|
API.
|
||||||
|
|
||||||
|
Each row also says how its model runs without being opened: replicas ready of
|
||||||
|
those it should have, how many are still starting, and the nodes they are on —
|
||||||
|
in amber when short, in red when parked or when it does not fit. It is one line
|
||||||
|
per model beside its mode, and nodes drop the part of the name every node
|
||||||
|
shares (node05 for llmgw-node05, the full name on hover).
|
||||||
|
|
||||||
|
### One line per record, in every table
|
||||||
|
|
||||||
|
The same rule across the admin pages. Headers do not wrap; a record is one
|
||||||
|
line; free text is cut to its column with the whole of it on hover rather than
|
||||||
|
given a row of its own; a row across the whole table is only a note or an open
|
||||||
|
form. On the cluster page that means a node is one line (its address and logs
|
||||||
|
beside its name, not under it), a model's description is a column rather than a
|
||||||
|
second row, node names drop the prefix they all share, and the GPUs, classes,
|
||||||
|
fabrics and fallbacks columns appear only when some row has something in them —
|
||||||
|
on a CPU-only cluster they were three columns of dashes pushing the table past
|
||||||
|
the page. On People, a person in many teams shows two and a count, the rest on
|
||||||
|
hover.
|
||||||
|
|
||||||
|
## v0.10.4 — 2026-09-30
|
||||||
|
|
||||||
|
Published first as release candidates v0.10.4-rc.1 to rc.9.
|
||||||
|
|
||||||
|
After upgrading, run `llmgw ctl restart --all` once: CPU replicas take the new
|
||||||
|
thread count below only when they start.
|
||||||
|
|
||||||
|
### CPU replicas no longer fight over the CPUs
|
||||||
|
|
||||||
|
llama.cpp computes on every CPU of the machine unless told otherwise, and the
|
||||||
|
agent did not tell it. Two busy CPU replicas on one node then spin against each
|
||||||
|
other: on an 8-vCPU node, 7B generation fell from 10 tokens a second alone to
|
||||||
|
0.8 each with both busy, in two rounds of the same test. The agent now starts a
|
||||||
|
CPU replica on half the machine's CPUs, where one alone makes 6.5 and two busy
|
||||||
|
make 5.5 each. `llamacpp.threads` in the agent's configuration sets it for a
|
||||||
|
node — all of them suits a node that only ever runs one CPU replica — and a
|
||||||
|
profile's own `--threads` still wins. Measured on VMs whose host is
|
||||||
|
oversubscribed; the proportions, not the numbers, are the point.
|
||||||
|
|
||||||
|
### Tracing
|
||||||
|
|
||||||
|
With `tracing.endpoint` set (or `OTEL_EXPORTER_OTLP_ENDPOINT`), each inference
|
||||||
|
request is traced over OTLP/HTTP: a span for the request, carrying its
|
||||||
|
`X-Request-Id`, with admission, scheduling and each attempt at an engine under
|
||||||
|
it. A caller's own `traceparent` is continued, and the engine is sent one for
|
||||||
|
its attempt. `tracing.sample_ratio` traces a share of the requests that arrive
|
||||||
|
without a decision. Off, it costs nothing.
|
||||||
|
|
||||||
|
### Restarting replicas
|
||||||
|
|
||||||
|
`llmgw ctl restart REPLICA... | --model M | --all` replaces replicas with new
|
||||||
|
ones. Each replacement is started beside the replica it replaces, which keeps
|
||||||
|
serving until the new one is ready; replicas of one model go one at a time.
|
||||||
|
Where no second copy fits anywhere, the old replica is stopped after the drain
|
||||||
|
grace period and the new one starts in its place. This is how running engines
|
||||||
|
pick up the keys introduced in v0.10.3, an edited profile, or a new engine
|
||||||
|
version.
|
||||||
|
|
||||||
|
### A model that keeps failing is parked, and everybody is told
|
||||||
|
|
||||||
|
A model whose replicas failed at every start was retried forever: kev-0.5b,
|
||||||
|
whose architecture the nodes' llama.cpp did not know, was started 155 times in
|
||||||
|
eight hours. After five failures in a row a model is now parked — not placed
|
||||||
|
again until its definition changes or `llmgw ctl retry MODEL` is run — and
|
||||||
|
`ctl status`, `ctl models` and the admin API say so with the last error.
|
||||||
|
|
||||||
|
A request for a model that will not come up — parked, or fitting on no node —
|
||||||
|
is answered at once with the reason, instead of "the model is loading; retry
|
||||||
|
shortly", which a client would retry forever. Each request still makes the
|
||||||
|
control plane try placing it afresh, so a model that fits once room is made is
|
||||||
|
not refused on an old answer. "no placement for model" is logged when its
|
||||||
|
reason changes or every ten minutes, not on every pass: it was 250 identical
|
||||||
|
lines in two days for one model.
|
||||||
|
|
||||||
|
### Metrics and alerts for what goes wrong
|
||||||
|
|
||||||
|
New series: `llmgw_model_replicas_desired`, `llmgw_model_start_failures`,
|
||||||
|
`llmgw_model_parked`, `llmgw_model_unplaceable`, `llmgw_node_engine_install`,
|
||||||
|
`llmgw_control_plane_leader` and `llmgw_certificate_expiry_timestamp_seconds`.
|
||||||
|
`llmgw_model_replicas_ready` is now published at 0 for every model that should
|
||||||
|
be running; it was published only while a replica was ready, so
|
||||||
|
ModelHasNothingServing could never fire. New alerts: ModelParked,
|
||||||
|
ModelFailingToStart, ModelCannotBePlaced, EngineInstallFailed,
|
||||||
|
ControlPlaneLeadership, CertificateExpiresSoon and CertificateExpiresVerySoon.
|
||||||
|
|
||||||
|
`ctl status` counts a model against the replicas it should have now rather
|
||||||
|
than its minimum, so an on-demand model that was asked for and fitted nowhere
|
||||||
|
no longer reads as satisfied.
|
||||||
|
|
||||||
|
### Backups
|
||||||
|
|
||||||
|
`llmgw server backup` writes the store and `/etc/llmgw` — configuration,
|
||||||
|
secrets, and the certificate authority's key — to one archive readable by root
|
||||||
|
only, keeping the newest 14. `deploy/systemd/llmgw-backup.timer` runs it
|
||||||
|
nightly. The control plane publishes the newest archive's time as
|
||||||
|
`llmgw_backup_last_success_timestamp_seconds`, and BackupTooOld fires when it is
|
||||||
|
more than 26 hours old or there has never been one. docs/install.md now says how
|
||||||
|
to restore, including why running engines survive it: the engine-key secret
|
||||||
|
comes back with the store.
|
||||||
|
|
||||||
|
### Replicas follow load between min and max
|
||||||
|
|
||||||
|
A model's `max` was a ceiling nothing reached: every model ran exactly its
|
||||||
|
minimum however long its queue. Now, when requests are waiting at the engines —
|
||||||
|
every slot of every replica busy — at one or more per replica for a minute, one
|
||||||
|
replica is added, up to `max`; ten minutes with nothing waiting takes one of
|
||||||
|
those away, down to `min`. One step at a time, and not while the previous one
|
||||||
|
is still starting, so a spike that is over before a replica loads does not
|
||||||
|
leave several behind. Each decision is logged with the load that made it.
|
||||||
|
`llmgw ctl scale MODEL --min 1 --max 3` turns it on for a model.
|
||||||
|
|
||||||
|
### A model from the registry in one command
|
||||||
|
|
||||||
|
`llmgw ctl model serve OWNER/REPO` takes a model from the registry to a replica
|
||||||
|
that answers. It picks the GGUF file (`--quant`, Q4_K_M by default), has the
|
||||||
|
mirror fetch it and checks what arrived against the registry's published
|
||||||
|
sha256, checks that the nodes' llama.cpp can load it, sizes its memory from its
|
||||||
|
header, adds it to the catalog, waits for a replica — stopping early with the
|
||||||
|
reason if it fits nowhere or keeps failing — and asks it to say hello. Each step
|
||||||
|
is shown as it happens. It runs in the control plane as a job, so a download of
|
||||||
|
hours does not depend on a terminal staying open. `--on-demand` adds it without
|
||||||
|
starting a replica.
|
||||||
|
|
||||||
|
### Upgrading from one command
|
||||||
|
|
||||||
|
`sudo llmgw ctl upgrade VERSION`, run on the control plane, upgrades that
|
||||||
|
machine and then every node in turn. Each agent is told the version over its
|
||||||
|
existing stream, fetches the release for its own platform, checks it against
|
||||||
|
the release's checksums, keeps the old binary beside the new and restarts,
|
||||||
|
while its engines keep serving. A node that does not come back on the new
|
||||||
|
version, ready, with its replicas, stops the upgrade before the next one.
|
||||||
|
Running it again resumes. Agents from before this have to be moved by hand
|
||||||
|
once.
|
||||||
|
|
||||||
|
### Following a request
|
||||||
|
|
||||||
|
Every response carries `X-Request-Id` — the client's own when it sends a plain
|
||||||
|
one, otherwise one the gateway makes — and the engine receives it too. One
|
||||||
|
access log line per request records it with the model asked for and served,
|
||||||
|
the replica, status, time queued, time to the first token, total time,
|
||||||
|
tokens, and the key, project and user. Usage records keep the id, replica,
|
||||||
|
queue time and time to first token, and `llmgw ctl request ID` reads one back:
|
||||||
|
the answer to "why was this slow?".
|
||||||
|
|
||||||
|
### Rate-limit headers
|
||||||
|
|
||||||
|
Responses carry OpenAI's `x-ratelimit-limit-*`, `x-ratelimit-remaining-*` and
|
||||||
|
`x-ratelimit-reset-*` for requests and tokens, from whichever level of the
|
||||||
|
tenancy tree will refuse first, so SDKs and editors can slow down before they
|
||||||
|
are refused.
|
||||||
|
|
||||||
|
### Memory sized from the model
|
||||||
|
|
||||||
|
The mirror's serve form sizes a llama.cpp model's memory from its GGUF header:
|
||||||
|
weights, plus the KV cache for the context pool from its layers, KV heads and
|
||||||
|
head width, plus the engine's own. At a 16k pool that cache ranges from 0.44 GB
|
||||||
|
to 5.25 GB across the models on the live mirror, which the old "weights plus
|
||||||
|
half again plus two" could not see.
|
||||||
|
|
||||||
|
### Security
|
||||||
|
|
||||||
|
- **Alertmanager answered anyone on the network**, including requests to
|
||||||
|
silence alerts. The monitoring README now binds it to loopback, where
|
||||||
|
Prometheus beside it is its only client.
|
||||||
|
- **Login and signup could be made to use gigabytes.** Each attempt hashes with
|
||||||
|
64 MiB of argon2, with no bound on how many ran at once, and the per-address
|
||||||
|
throttle grew by one entry per address tried. Hashes now run a few at a time
|
||||||
|
and the rest wait; the throttle expires old entries, and when it is still
|
||||||
|
full it refuses new addresses rather than dropping live ones — which would
|
||||||
|
have let a flood of made-up addresses reset the throttle on a real one.
|
||||||
|
- **Any key holder could grow Prometheus without bound** by asking for made-up
|
||||||
|
model names, each of which became a metric label. Requests are counted under
|
||||||
|
the catalog's name for their model, or "unknown".
|
||||||
|
- **A SQLite backup left a full copy of the store readable** by local users
|
||||||
|
while it ran. The snapshot is now made in a private directory, and
|
||||||
|
llmgw-backup.service sets `UMask=0077`.
|
||||||
|
- **An engine archive could write outside its directory through chained
|
||||||
|
links.** Archives are unpacked through `os.Root`, and a link may only point
|
||||||
|
at a file beside it. Every llama.cpp b10991 archive, CPU and CUDA, already
|
||||||
|
follows that rule.
|
||||||
|
- **Onboarding made every node trust the cluster's root for everything.** It
|
||||||
|
was added to the system store, so a certificate the control plane signed
|
||||||
|
would pass for any site with every TLS client on the node. The agent trusts
|
||||||
|
it through its own configuration only; onboarding no longer adds it, and
|
||||||
|
takes it out of a node that an older version onboarded.
|
||||||
|
- **Any node could manage a mirror running mutual TLS**, since one root signs
|
||||||
|
every node's certificate. `llmgw mirror serve --api-client NAME` answers only
|
||||||
|
the certificate with that common name, and the mirror warns when mutual TLS
|
||||||
|
runs without one.
|
||||||
|
- **The administrator's password stayed in `init.log`.** It is shown once, on
|
||||||
|
the screen, and the log keeps a placeholder.
|
||||||
|
- **Auth off on a public address was silent.** The server now warns at start
|
||||||
|
when the API listens beyond loopback without keys.
|
||||||
|
- **The actor recorded with a change was whatever the caller sent**, line
|
||||||
|
breaks included. It is cut to a short printable name, and `sudo llmgw ctl`
|
||||||
|
records the person who ran sudo rather than root.
|
||||||
|
- **vLLM's Ray head ran its dashboard**, whose job API runs code for anyone who
|
||||||
|
reaches it. It is off. The head itself still has no authentication, and
|
||||||
|
SGLang still takes its key as a flag; docs/install.md says what to do about
|
||||||
|
both. The monitoring README says how to keep Prometheus's queries to the
|
||||||
|
control plane.
|
||||||
|
|
||||||
|
### Fixes
|
||||||
|
|
||||||
|
- Tests that slept a fixed time and then asserted now wait for their
|
||||||
|
condition. They are the likeliest cause of CI's intermittent failures on a
|
||||||
|
loaded runner.
|
||||||
|
- `ctl` reported a change the API had queued (202 Accepted) as an error, as
|
||||||
|
`ctl restart` did on its first real use.
|
||||||
|
- The status each failure is answered with — 499, 502, 503 with and without
|
||||||
|
Retry-After, 504, 429 — is now tested case by case.
|
||||||
|
|
||||||
|
### Releases
|
||||||
|
|
||||||
|
Small fixes are published as release candidates of the next version —
|
||||||
|
v0.10.4-rc.1, rc.2 — which sort after v0.10.3 and before v0.10.4. Their release
|
||||||
|
notes come from this section, and they are marked as pre-releases. Onboarding
|
||||||
|
treats a release candidate as a release and downloads it.
|
||||||
|
|
||||||
|
## v0.10.3 — 2026-09-30
|
||||||
|
|
||||||
|
Security fixes from a review of v0.10.2. Three of them were confirmed on the
|
||||||
|
live cluster before being fixed.
|
||||||
|
|
||||||
|
### Engines demand a key only the router holds
|
||||||
|
|
||||||
|
Engines listened on every address of their node with no key. A request from a
|
||||||
|
neighbouring node reached the model directly, past the gateway's
|
||||||
|
authentication, quotas and accounting. Each replica's engine is now started
|
||||||
|
with a key derived from the replica id and a secret in the store. The router,
|
||||||
|
its metrics scrape, the agent's readiness check and `ctl bench` send it.
|
||||||
|
llama.cpp and vLLM receive it through their environment, not their command
|
||||||
|
line. Engines started before this keep running without a key until their
|
||||||
|
replica is next started.
|
||||||
|
|
||||||
|
### The mirror's management API answers only who it was told to
|
||||||
|
|
||||||
|
Without `--api-allow`, the management API answered anyone who could reach it,
|
||||||
|
including every node. It now answers this machine only; the control plane is
|
||||||
|
let in with `--api-allow` or mutual TLS. **Upgrading a mirror set up by hand:
|
||||||
|
add `--api-allow <control plane address>` to its unit.** Downloads started
|
||||||
|
through the API go to http and https only, and never to loopback, private or
|
||||||
|
link-local addresses (`--allow-private-sources` to permit that). Model,
|
||||||
|
version and file names can no longer be `..` or start with a dot.
|
||||||
|
|
||||||
|
### The admin listener refuses what a browser page could send
|
||||||
|
|
||||||
|
The admin API has no login and trusts whoever reaches it. A page open in an
|
||||||
|
operator's browser, reaching it over an ssh tunnel, could post a model
|
||||||
|
definition. The listener now requires Host to be an address or localhost,
|
||||||
|
refuses cross-site requests, and takes changes only as `application/json`.
|
||||||
|
A profile's `env` may no longer set variables that make an engine run other
|
||||||
|
code (`LD_*`, `PATH`, `PYTHON*` start-up variables, `BASH_ENV` and similar).
|
||||||
|
They are refused where a model is defined and dropped again before an engine
|
||||||
|
starts.
|
||||||
|
|
||||||
|
### Fixes
|
||||||
|
|
||||||
|
- The agent installs only engine versions that look like one, whatever the
|
||||||
|
control plane sent; a version of `../x` could have installed outside
|
||||||
|
`/opt/llmgw/engines`.
|
||||||
|
|
||||||
## v0.10.2 — 2026-09-29
|
## v0.10.2 — 2026-09-29
|
||||||
|
|
||||||
- After `server init`'s screen closes, only the administrator's password and
|
- After `server init`'s screen closes, only the administrator's password and
|
||||||
|
|||||||
+609
-171
File diff suppressed because it is too large.
Load diff
@@ -26,6 +26,7 @@ message AgentMessage {
|
|||||||
LogsReply logs = 4;
|
LogsReply logs = 4;
|
||||||
FabricScanReply fabric_scan = 5;
|
FabricScanReply fabric_scan = 5;
|
||||||
FabricTestReply fabric_test = 6;
|
FabricTestReply fabric_test = 6;
|
||||||
|
UpgradeReply upgrade = 7;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -41,6 +42,7 @@ message ControlMessage {
|
|||||||
LogsRequest logs = 7;
|
LogsRequest logs = 7;
|
||||||
FabricScanRequest fabric_scan = 8;
|
FabricScanRequest fabric_scan = 8;
|
||||||
FabricTestRequest fabric_test = 9;
|
FabricTestRequest fabric_test = 9;
|
||||||
|
UpgradeAgent upgrade = 10;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,6 +120,10 @@ message Register {
|
|||||||
// meant "an agent too old to say" and constrained nothing — so a freshly
|
// meant "an agent too old to say" and constrained nothing — so a freshly
|
||||||
// built node with no engine installed yet was a candidate for every model.
|
// built node with no engine installed yet was a candidate for every model.
|
||||||
bool engines_probed = 14;
|
bool engines_probed = 14;
|
||||||
|
// cpu_count is the CPUs the agent may run on. Placement reserves cores
|
||||||
|
// for CPU replicas out of it, the way it reserves memory out of
|
||||||
|
// memory_total_mb; zero, from an older agent, reserves nothing.
|
||||||
|
int32 cpu_count = 18;
|
||||||
// imex is the service that lets a multi-node NVLink domain share memory.
|
// imex is the service that lets a multi-node NVLink domain share memory.
|
||||||
IMEX imex = 15;
|
IMEX imex = 15;
|
||||||
|
|
||||||
@@ -279,6 +285,25 @@ message NodeLoad {
|
|||||||
int64 memory_used_bytes = 6;
|
int64 memory_used_bytes = 6;
|
||||||
int64 memory_available_bytes = 7;
|
int64 memory_available_bytes = 7;
|
||||||
repeated ShardLoad shards = 8;
|
repeated ShardLoad shards = 8;
|
||||||
|
// What the machine's own sensors say; zero is not measured. cpu_temp_c is
|
||||||
|
// the hottest CPU sensor, cpu_power_w the CPU packages' draw from RAPL,
|
||||||
|
// inlet_temp_c the board's inlet or ambient sensor where it has one.
|
||||||
|
double cpu_temp_c = 9;
|
||||||
|
double cpu_power_w = 10;
|
||||||
|
double inlet_temp_c = 11;
|
||||||
|
repeated GPUSensor gpus = 12;
|
||||||
|
}
|
||||||
|
|
||||||
|
// GPUSensor is one card's heat, power, and what is holding it back.
|
||||||
|
message GPUSensor {
|
||||||
|
int32 index = 1;
|
||||||
|
double temp_c = 2;
|
||||||
|
double memory_temp_c = 3;
|
||||||
|
double power_w = 4;
|
||||||
|
double power_limit_w = 5;
|
||||||
|
// throttle names what is holding the clocks down now: sw_power_cap,
|
||||||
|
// hw_slowdown, sw_thermal, hw_thermal, power_brake.
|
||||||
|
repeated string throttle = 6;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ShardLoad is one engine process group's use.
|
// ShardLoad is one engine process group's use.
|
||||||
@@ -418,6 +443,8 @@ message Heartbeat {
|
|||||||
// domain, and the node has to stop being offered work that spans it.
|
// domain, and the node has to stop being offered work that spans it.
|
||||||
repeated GPUFabricState gpu_fabric = 16;
|
repeated GPUFabricState gpu_fabric = 16;
|
||||||
IMEX imex = 17;
|
IMEX imex = 17;
|
||||||
|
// cpu_count as in Register, for a VM given more CPUs while it runs.
|
||||||
|
int32 cpu_count = 18;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Provision is the software a node should have: which engines, at which
|
// Provision is the software a node should have: which engines, at which
|
||||||
@@ -524,6 +551,13 @@ message StartShard {
|
|||||||
// profile_fp identifies the version of the profile, so the agent can say
|
// profile_fp identifies the version of the profile, so the agent can say
|
||||||
// which one a running shard was started from. The agent never interprets it.
|
// which one a running shard was started from. The agent never interprets it.
|
||||||
string profile_fp = 15;
|
string profile_fp = 15;
|
||||||
|
// api_key is what the engine demands of every request, derived by the
|
||||||
|
// control plane from the replica id. The router sends it; nothing else
|
||||||
|
// that can reach the node's ports has it.
|
||||||
|
string api_key = 16;
|
||||||
|
// cpu_threads is how many CPU threads a shard with no GPU computes on:
|
||||||
|
// the cores placement reserved for it. Zero leaves it to the agent.
|
||||||
|
int32 cpu_threads = 17;
|
||||||
}
|
}
|
||||||
|
|
||||||
message EngineProfile {
|
message EngineProfile {
|
||||||
@@ -538,6 +572,9 @@ message EngineProfile {
|
|||||||
map<string, string> env = 9;
|
map<string, string> env = 9;
|
||||||
string model_path = 10;
|
string model_path = 10;
|
||||||
string served_model_name = 11;
|
string served_model_name = 11;
|
||||||
|
// adapters are LoRA adapters loaded beside the model, in order: the index
|
||||||
|
// is the id a request applies one by.
|
||||||
|
repeated LoraAdapter adapters = 12;
|
||||||
}
|
}
|
||||||
|
|
||||||
// RemoveWeights deletes cached model files a node no longer needs.
|
// RemoveWeights deletes cached model files a node no longer needs.
|
||||||
@@ -645,3 +682,35 @@ message FabricTestReply {
|
|||||||
string error = 4;
|
string error = 4;
|
||||||
int64 took_ms = 5;
|
int64 took_ms = 5;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UpgradeAgent tells an agent to replace its own binary with a release and
|
||||||
|
// restart.
|
||||||
|
//
|
||||||
|
// The agent fetches the release for its own platform and checks it against
|
||||||
|
// the release's checksums before replacing anything, keeps the old binary
|
||||||
|
// beside the new, then exits so systemd starts the new one. Engines keep
|
||||||
|
// running throughout, as for any agent restart, and are adopted on the way
|
||||||
|
// back. Over the stream the agent already holds, so upgrading a fleet needs
|
||||||
|
// no ssh into it.
|
||||||
|
message UpgradeAgent {
|
||||||
|
string request_id = 1;
|
||||||
|
// version must be a release: v0.10.4 or v0.10.4-rc.2.
|
||||||
|
string version = 2;
|
||||||
|
// releases is the base URL release files are downloaded from.
|
||||||
|
string releases = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpgradeReply says whether the new binary is in place. Sent before the
|
||||||
|
// agent exits; an empty error means it will be back on the new version.
|
||||||
|
message UpgradeReply {
|
||||||
|
string request_id = 1;
|
||||||
|
string version = 2;
|
||||||
|
string error = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoraAdapter is one adapter: the name clients ask for it by, and its source,
|
||||||
|
// which the agent fetches as it does the model's weights.
|
||||||
|
message LoraAdapter {
|
||||||
|
string name = 1;
|
||||||
|
string source = 2;
|
||||||
|
}
|
||||||
@@ -122,9 +122,10 @@ TMP="$(mktemp -d)"
|
|||||||
trap 'rm -rf "$TMP"' EXIT
|
trap 'rm -rf "$TMP"' EXIT
|
||||||
"$BINARY" ctl cert issue-client "$NODE" --dir "$PKI" --out "$TMP" >/dev/null
|
"$BINARY" ctl cert issue-client "$NODE" --dir "$PKI" --out "$TMP" >/dev/null
|
||||||
|
|
||||||
# 2 and 3. Identity, and trusting the root so future tooling on that machine
|
# 2 and 3. Identity. The root is trusted by the agent's configuration only,
|
||||||
# does not treat our certificates as strangers.
|
# never system-wide: it signs whatever the control plane asks, and every
|
||||||
say "installing the certificate and trusting the root"
|
# TLS client on the machine would take it for any site.
|
||||||
|
say "installing the certificate"
|
||||||
scp -q $SSHOPTS "$PKI/ca.pem" "$TMP/$NODE.pem" "$TMP/$NODE-key.pem" "$SSH_USER@$ADDR:/tmp/"
|
scp -q $SSHOPTS "$PKI/ca.pem" "$TMP/$NODE.pem" "$TMP/$NODE-key.pem" "$SSH_USER@$ADDR:/tmp/"
|
||||||
$SSH "set -e
|
$SSH "set -e
|
||||||
sudo install -d -m 0755 /etc/llmgw
|
sudo install -d -m 0755 /etc/llmgw
|
||||||
@@ -132,8 +133,10 @@ $SSH "set -e
|
|||||||
sudo install -m 0644 /tmp/$NODE.pem /etc/llmgw/agent.pem
|
sudo install -m 0644 /tmp/$NODE.pem /etc/llmgw/agent.pem
|
||||||
sudo install -m 0600 /tmp/$NODE-key.pem /etc/llmgw/agent-key.pem
|
sudo install -m 0600 /tmp/$NODE-key.pem /etc/llmgw/agent-key.pem
|
||||||
sudo chown root:root /etc/llmgw/*
|
sudo chown root:root /etc/llmgw/*
|
||||||
sudo install -m 0644 /tmp/ca.pem /usr/local/share/ca-certificates/llmgw-ca.crt
|
if [ -f /usr/local/share/ca-certificates/llmgw-ca.crt ]; then
|
||||||
sudo update-ca-certificates >/dev/null 2>&1 || true
|
sudo rm -f /usr/local/share/ca-certificates/llmgw-ca.crt
|
||||||
|
sudo update-ca-certificates --fresh >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
rm -f /tmp/ca.pem /tmp/$NODE.pem /tmp/$NODE-key.pem"
|
rm -f /tmp/ca.pem /tmp/$NODE.pem /tmp/$NODE-key.pem"
|
||||||
|
|
||||||
# 4. The binary, the configuration and the unit.
|
# 4. The binary, the configuration and the unit.
|
||||||
@@ -143,7 +146,7 @@ $SSH "set -e
|
|||||||
# that is not a release (a developer's, with commits past the tag) has no such
|
# that is not a release (a developer's, with commits past the tag) has no such
|
||||||
# file, and is copied as before.
|
# file, and is copied as before.
|
||||||
TAG="$("$BINARY" --version 2>/dev/null | awk '{print $NF}')"
|
TAG="$("$BINARY" --version 2>/dev/null | awk '{print $NF}')"
|
||||||
if printf '%s' "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
|
if printf '%s' "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$'; then
|
||||||
say "installing llmgw $TAG from the release"
|
say "installing llmgw $TAG from the release"
|
||||||
$SSH "set -e
|
$SSH "set -e
|
||||||
arch=\$(dpkg --print-architecture)
|
arch=\$(dpkg --print-architecture)
|
||||||
|
|||||||
@@ -46,6 +46,19 @@ apt-get install prometheus prometheus-alertmanager
|
|||||||
install -m 0644 prometheus.yml /etc/prometheus/prometheus.yml
|
install -m 0644 prometheus.yml /etc/prometheus/prometheus.yml
|
||||||
install -m 0644 llmgw-rules.yml /etc/prometheus/rules/llmgw.yml
|
install -m 0644 llmgw-rules.yml /etc/prometheus/rules/llmgw.yml
|
||||||
install -m 0644 alertmanager.yml /etc/prometheus/alertmanager.yml
|
install -m 0644 alertmanager.yml /etc/prometheus/alertmanager.yml
|
||||||
|
# Alertmanager answers on loopback only. Its API takes silences from anyone who
|
||||||
|
# can reach it, with no login, and silencing BackupTooOld or ModelParked is how
|
||||||
|
# something that should be noticed is not. Prometheus beside it is its only
|
||||||
|
# client; one Alertmanager needs no cluster port.
|
||||||
|
echo 'ARGS="--web.listen-address=127.0.0.1:9093 --cluster.listen-address="' \
|
||||||
|
> /etc/default/prometheus-alertmanager
|
||||||
|
# Prometheus itself is read-only as installed: its admin and lifecycle APIs
|
||||||
|
# stay off (no --web.enable-admin-api, no --web.enable-lifecycle). It still
|
||||||
|
# answers any query from anyone who reaches :9090, which says which models and
|
||||||
|
# nodes there are and how busy each is. Its readers are the control plane
|
||||||
|
# (monitoring.prometheus) and Grafana beside it; to keep it to those, allow
|
||||||
|
# the control plane and drop the rest, e.g. with nftables:
|
||||||
|
# nft add rule inet filter input tcp dport 9090 ip saddr != { 127.0.0.1, CONTROL_PLANE_IP } drop
|
||||||
promtool check config /etc/prometheus/prometheus.yml
|
promtool check config /etc/prometheus/prometheus.yml
|
||||||
systemctl restart prometheus prometheus-alertmanager
|
systemctl restart prometheus prometheus-alertmanager
|
||||||
|
|
||||||
@@ -75,7 +88,8 @@ Alertmanager groups and holds alerts and has no receiver.
|
|||||||
Where an alert should go — an address, a Slack workspace, a webhook — is a
|
Where an alert should go — an address, a Slack workspace, a webhook — is a
|
||||||
decision with credentials attached, and a monitoring system that quietly starts
|
decision with credentials attached, and a monitoring system that quietly starts
|
||||||
mailing people is worse than one that is visibly not wired up. Alerts are
|
mailing people is worse than one that is visibly not wired up. Alerts are
|
||||||
visible at `:9093` and in Prometheus; add a receiver to `alertmanager.yml` to
|
visible in Prometheus, and at `:9093` on the monitoring host itself (`ssh -L
|
||||||
|
9093:127.0.0.1:9093 <host>` to open its page); add a receiver to `alertmanager.yml` to
|
||||||
make them travel. `critical` already routes separately so it can be pointed
|
make them travel. `critical` already routes separately so it can be pointed
|
||||||
somewhere else without touching the rest.
|
somewhere else without touching the rest.
|
||||||
|
|
||||||
@@ -86,7 +100,7 @@ An alerting system nobody has seen fire is an assumption.
|
|||||||
```
|
```
|
||||||
llmgw ctl cordon <node> # work already there keeps running
|
llmgw ctl cordon <node> # work already there keeps running
|
||||||
# ~2 minutes later
|
# ~2 minutes later
|
||||||
curl -s http://<host>:9093/api/v2/alerts | jq '.[].labels'
|
ssh <host> curl -s http://127.0.0.1:9093/api/v2/alerts | jq '.[].labels'
|
||||||
llmgw ctl uncordon <node> # the alert resolves on its own
|
llmgw ctl uncordon <node> # the alert resolves on its own
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -42,10 +42,119 @@ groups:
|
|||||||
annotations:
|
annotations:
|
||||||
summary: "{{ $labels.model }} has no replica ready"
|
summary: "{{ $labels.model }} has no replica ready"
|
||||||
description: >-
|
description: >-
|
||||||
The model is placed and nothing is serving it. Ten minutes, because
|
The model should be running and nothing is serving it. Ten minutes,
|
||||||
a large model on CPU legitimately takes minutes to load. A model
|
because a large model on CPU legitimately takes minutes to load. The
|
||||||
that is deliberately stopped publishes no series at all, so this
|
series is published at 0 for every model that should be running —
|
||||||
cannot fire for one nobody asked for.
|
before v0.10.4 it was published only while a replica was ready, so
|
||||||
|
this could not fire at all. A model deliberately stopped, or an
|
||||||
|
on-demand one nobody is asking for, publishes nothing.
|
||||||
|
|
||||||
|
- alert: ModelParked
|
||||||
|
expr: llmgw_model_parked == 1
|
||||||
|
labels: { severity: critical }
|
||||||
|
annotations:
|
||||||
|
summary: "{{ $labels.model }} failed every time it started and is no longer retried"
|
||||||
|
description: >-
|
||||||
|
Five replicas in a row failed at start. Before this existed a model
|
||||||
|
whose weights its engine could not load was started 155 times over
|
||||||
|
eight hours. `llmgw ctl status` has the last error; once it is dealt
|
||||||
|
with, `llmgw ctl retry <model>`. Changing the model's definition
|
||||||
|
also retries it.
|
||||||
|
|
||||||
|
- alert: ModelFailingToStart
|
||||||
|
expr: llmgw_model_start_failures >= 2
|
||||||
|
for: 5m
|
||||||
|
labels: { severity: warning }
|
||||||
|
annotations:
|
||||||
|
summary: "{{ $labels.model }} has failed to start {{ $value }} times in a row"
|
||||||
|
description: >-
|
||||||
|
On its way to being parked. The engine's own words are in
|
||||||
|
`llmgw ctl logs <node> --source <shard>` for the next attempt, or in
|
||||||
|
the control plane's log for the last one.
|
||||||
|
|
||||||
|
- alert: ModelCannotBePlaced
|
||||||
|
expr: llmgw_model_unplaceable == 1
|
||||||
|
for: 10m
|
||||||
|
labels: { severity: warning }
|
||||||
|
annotations:
|
||||||
|
summary: "{{ $labels.model }} is wanted and fits on no node"
|
||||||
|
description: >-
|
||||||
|
Short of replicas, and the last attempt found nowhere with the
|
||||||
|
memory, cards or engine it needs. `llmgw ctl explain-placement
|
||||||
|
<model>` says what each node lacks. For an on-demand model, requests
|
||||||
|
for it are told this rather than asked to retry.
|
||||||
|
|
||||||
|
- alert: ModelPreempted
|
||||||
|
expr: increase(llmgw_model_preempted_total[15m]) > 0
|
||||||
|
labels: { severity: info }
|
||||||
|
annotations:
|
||||||
|
summary: "{{ $labels.model }} ({{ $labels.class }}) gave up replicas to a higher class"
|
||||||
|
description: >-
|
||||||
|
Capacity ran short and a model of a higher class took this one's
|
||||||
|
room. Expected now and then; if it keeps firing for the same model,
|
||||||
|
the cluster is too small for what is asked of it at these classes.
|
||||||
|
`llmgw ctl models` shows who it gave way to.
|
||||||
|
|
||||||
|
- alert: CapacityAtLimit
|
||||||
|
expr: llmgw_capacity_holds >= on(scope, resource) llmgw_capacity_limit
|
||||||
|
for: 15m
|
||||||
|
labels: { severity: info }
|
||||||
|
annotations:
|
||||||
|
summary: "{{ $labels.scope }} holds all the {{ $labels.resource }} it may"
|
||||||
|
description: >-
|
||||||
|
Its models get no further replicas, under load or not, until
|
||||||
|
something of theirs stops or the limit is raised: `llmgw ctl quota
|
||||||
|
list`, `llmgw ctl quota set`.
|
||||||
|
|
||||||
|
- alert: EngineInstallFailed
|
||||||
|
expr: llmgw_node_engine_install{state="failed"} == 1
|
||||||
|
for: 15m
|
||||||
|
labels: { severity: warning }
|
||||||
|
annotations:
|
||||||
|
summary: "{{ $labels.engine }} {{ $labels.version }} will not install on {{ $labels.node }}"
|
||||||
|
description: >-
|
||||||
|
The node takes no model needing it until it does. The agent retries
|
||||||
|
on its own; `llmgw ctl nodes` and the node's logs on the cluster
|
||||||
|
page say why it failed.
|
||||||
|
|
||||||
|
- alert: ControlPlaneLeadership
|
||||||
|
expr: sum(llmgw_control_plane_leader) != 1
|
||||||
|
for: 2m
|
||||||
|
labels: { severity: critical }
|
||||||
|
annotations:
|
||||||
|
summary: "{{ $value }} control planes believe they are placing work"
|
||||||
|
description: >-
|
||||||
|
None means nothing is placed or replaced; two means both are, each
|
||||||
|
into GPUs the other believes free. The lease is in the store: check
|
||||||
|
it is reachable from every control plane.
|
||||||
|
|
||||||
|
- alert: CertificateExpiresSoon
|
||||||
|
expr: (llmgw_certificate_expiry_timestamp_seconds - time()) < 30 * 86400
|
||||||
|
labels: { severity: warning }
|
||||||
|
annotations:
|
||||||
|
summary: "{{ $labels.cert }} ({{ $labels.subject }}) expires in {{ $value | humanizeDuration }}"
|
||||||
|
description: >-
|
||||||
|
Certificates last a year and nothing renews them. When the agent
|
||||||
|
server's runs out every agent refuses to connect at once. Reissue
|
||||||
|
with `llmgw ctl cert` and restart the control plane.
|
||||||
|
|
||||||
|
- alert: CertificateExpiresVerySoon
|
||||||
|
expr: (llmgw_certificate_expiry_timestamp_seconds - time()) < 7 * 86400
|
||||||
|
labels: { severity: critical }
|
||||||
|
annotations:
|
||||||
|
summary: "{{ $labels.cert }} ({{ $labels.subject }}) expires in {{ $value | humanizeDuration }}"
|
||||||
|
description: See CertificateExpiresSoon; a week is not long.
|
||||||
|
|
||||||
|
- alert: BackupTooOld
|
||||||
|
expr: time() - llmgw_backup_last_success_timestamp_seconds > 26 * 3600
|
||||||
|
labels: { severity: warning }
|
||||||
|
annotations:
|
||||||
|
summary: "The newest llmgw backup is {{ $value | humanizeDuration }} old"
|
||||||
|
description: >-
|
||||||
|
llmgw-backup.timer runs nightly. Also fires when there has never
|
||||||
|
been a backup (the series is 0 then). `journalctl -u
|
||||||
|
llmgw-backup.service` on the control plane says why the last one
|
||||||
|
failed; docs/install.md, "Backing up", says how to set it up.
|
||||||
|
|
||||||
- alert: PoolSaturated
|
- alert: PoolSaturated
|
||||||
expr: llmgw_pool_saturation > 1
|
expr: llmgw_pool_saturation > 1
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=llmgw backup: the store and /etc/llmgw, to /var/backups/llmgw-server
|
||||||
|
After=llmgw-server.service postgresql.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
# Nothing it writes is for anyone else: the archive holds the CA's key and a
|
||||||
|
# copy of the store.
|
||||||
|
UMask=0077
|
||||||
|
# Root, because the archive includes /etc/llmgw/secrets and the certificate
|
||||||
|
# authority's key, which the llmgw user deliberately cannot read all of.
|
||||||
|
ExecStart=/usr/local/bin/llmgw server backup --config /etc/llmgw/server.yaml --keep 14
|
||||||
|
Nice=10
|
||||||
|
IOSchedulingClass=idle
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Back up llmgw every night
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*-*-* 03:15:00
|
||||||
|
# A machine that was off at 03:15 backs up when it next starts, rather than
|
||||||
|
# waiting another day with a gap nobody notices until the alert.
|
||||||
|
Persistent=true
|
||||||
|
RandomizedDelaySec=10m
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -0,0 +1,303 @@
|
|||||||
|
# Facility-aware placement: rack power, temperature and cooling
|
||||||
|
|
||||||
|
**Status: step 0, reading the nodes' own sensors, is built; steps 1–5 are
|
||||||
|
deferred** until the rack, PDU and power information of a real site is at
|
||||||
|
hand — without it they would be designed against guesses.
|
||||||
|
This is the design for step 5 of resource allocation, after reserved CPU cores
|
||||||
|
and memory (1), capacity classes and preemption (2), tenant capacity quotas (3)
|
||||||
|
and sizing by measured throughput (4), all of which shipped in v0.10.5. Of
|
||||||
|
what follows, only the node sensors (rollout step 0) exist in the code: they
|
||||||
|
are read, published and shown, and change no decision.
|
||||||
|
|
||||||
|
## Why
|
||||||
|
|
||||||
|
In a large installation the scarce thing is often not a GPU but the room it
|
||||||
|
sits in. Three failures are invisible to placement today:
|
||||||
|
|
||||||
|
- **A rack goes over its power budget.** Every node in it reports healthy;
|
||||||
|
placement sees free cards and fills them; the breaker, or the facility's
|
||||||
|
power cap, decides the rest. A rack with A and B feeds is worse: it runs
|
||||||
|
fine until one feed fails and the other cannot carry the load.
|
||||||
|
- **Hot hardware is slow hardware.** GPUs throttle long before they fail. A
|
||||||
|
replica placed in a rack whose inlet air is too warm serves fewer tokens per
|
||||||
|
second than its measured capacity says, and nothing connects the two.
|
||||||
|
- **Correlated failure.** Two replicas of a model on two nodes in the same rack
|
||||||
|
or on the same PDU are one failure away from zero. Placement spreads replicas
|
||||||
|
across nodes, not across racks.
|
||||||
|
|
||||||
|
The goal is for placement to treat facility limits the way it already treats
|
||||||
|
memory and cores: reserve against a budget, refuse what does not fit, say why,
|
||||||
|
and never let missing data stop the cluster.
|
||||||
|
|
||||||
|
## What it builds on
|
||||||
|
|
||||||
|
| Already in place | Used for |
|
||||||
|
|---|---|
|
||||||
|
| Node labels, reported by agents, matched by `node_selector` | Where a node is: its rack, row, PDU |
|
||||||
|
| `inventory.Allocation` carries GPUs, memory and cores per shard | Adding watts, summed per rack and PDU |
|
||||||
|
| Placement rejections (`explain-placement`, headroom's "why not") | Naming the rack that is full or hot |
|
||||||
|
| Headroom: the scheduler dry-run on a copy of the inventory | Room that respects power budgets, for free |
|
||||||
|
| Preemption with classes, minimums and a dry run (step 2) | Shedding load from a hot rack in the right order |
|
||||||
|
| Measured throughput per profile (step 4) | Tokens per watt as a placement preference |
|
||||||
|
| Node sensors, read by every agent (step 0, built) | Heat and power with no facility integration: see below |
|
||||||
|
| `internal/promq`, a Prometheus client | Reading facility metrics that DCIM exporters publish |
|
||||||
|
|
||||||
|
## Concepts
|
||||||
|
|
||||||
|
**Location.** Each node has a place in a tree: site, room, row, rack. Power has
|
||||||
|
its own tree beside it: a rack draws from one or more PDUs, and each PDU from a
|
||||||
|
feed. Both are read from node labels with well-known keys:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# agent.yaml
|
||||||
|
labels:
|
||||||
|
llmgw.io/site: sto1
|
||||||
|
llmgw.io/room: hall-b
|
||||||
|
llmgw.io/row: "12"
|
||||||
|
llmgw.io/rack: r12-07
|
||||||
|
llmgw.io/pdu: r12-07-a,r12-07-b # both feeds, for a dual-fed rack
|
||||||
|
```
|
||||||
|
|
||||||
|
A node without these labels is its own rack of one: placement behaves as it
|
||||||
|
does today.
|
||||||
|
|
||||||
|
**Domain.** A rack, a row, a PDU, a room: anything with a budget or a
|
||||||
|
condition. Domains are declared in a facility file (below) or discovered from
|
||||||
|
node labels; a label naming a domain the file does not declare creates it with
|
||||||
|
no budget and no condition.
|
||||||
|
|
||||||
|
**Budget.** A domain may have a power cap in watts. Placement reserves each
|
||||||
|
replica's estimated draw against every domain its node belongs to and refuses a
|
||||||
|
placement that would take any of them past its cap less a safety margin. For a
|
||||||
|
dual-fed rack the cap that matters is what one feed can carry alone, since that
|
||||||
|
is what the rack must survive.
|
||||||
|
|
||||||
|
**Condition.** A domain has a thermal and cooling condition: `ok`, `warm`, `hot`
|
||||||
|
or `unknown`, derived from readings against thresholds with hysteresis.
|
||||||
|
Conditions change what placement prefers and allows; they are not amounts to
|
||||||
|
fill.
|
||||||
|
|
||||||
|
**Reading.** A measured value with a time: inlet temperature, power draw, a
|
||||||
|
cooling unit's alarm state. Readings older than their staleness limit are
|
||||||
|
treated as absent.
|
||||||
|
|
||||||
|
## Estimating a replica's draw
|
||||||
|
|
||||||
|
What placement reserves has to be known before the replica runs, so it is an
|
||||||
|
estimate first and a measurement later:
|
||||||
|
|
||||||
|
1. **Declared.** A profile may say `power_watts`. The operator's figure wins.
|
||||||
|
2. **Derived.** Otherwise GPU profiles take each card's power limit from NVML
|
||||||
|
(the agent already reads the cards; the power limit is one more field) plus
|
||||||
|
a per-node share for CPU, memory and fans; CPU profiles take their reserved
|
||||||
|
cores times a per-core figure for the node's CPU model.
|
||||||
|
3. **Measured.** Once replicas run, the agent reports their cards' actual draw
|
||||||
|
(NVML) and, where the BMC allows, the node's (Redfish or IPMI DCMI). A
|
||||||
|
domain's headroom is computed from the larger of what is reserved and what
|
||||||
|
is measured, so an estimate that was too low is corrected by the room it
|
||||||
|
stops offering, not by an outage.
|
||||||
|
|
||||||
|
## Decisions
|
||||||
|
|
||||||
|
### Power
|
||||||
|
|
||||||
|
- **Hard limit.** A placement that would take any domain past `cap × (1 −
|
||||||
|
margin)` is rejected, with the domain and the figures: *rack r12-07: 14.2 of
|
||||||
|
16.0 kW reserved, replica needs ~2.8, margin 10%*.
|
||||||
|
- **Feeds.** For a domain with two feeds the cap is the smaller feed's rating
|
||||||
|
times its derating (typically 80%), not the sum.
|
||||||
|
- **Already over.** A domain found over its cap, because the estimate was low
|
||||||
|
or the cap was lowered, gets no new placements, and an alert. It is not
|
||||||
|
emptied automatically; shedding power is the facility's decision, and
|
||||||
|
stopping inference to do it is ours only if asked (see shedding below).
|
||||||
|
|
||||||
|
### Temperature and cooling
|
||||||
|
|
||||||
|
Thresholds are per domain with defaults, and each condition needs to hold for a
|
||||||
|
while before it changes, in both directions, so a single reading neither
|
||||||
|
starts nor ends anything:
|
||||||
|
|
||||||
|
| Condition | Enters when | Placement |
|
||||||
|
|---|---|---|
|
||||||
|
| `ok` | below the warm threshold for 5 min | normal |
|
||||||
|
| `warm` | inlet above the warm threshold for 2 min, GPU temperatures near throttling, or any GPU thermally throttled | allowed, scored down: other domains first |
|
||||||
|
| `hot` | inlet above the hot threshold for 2 min, a cooling unit in alarm, or a sustained share of the domain's GPUs thermally throttled | no new placements, as a cordon would, but automatic and reversible |
|
||||||
|
| `unknown` | no fresh reading | as `ok`, and reported as unknown |
|
||||||
|
|
||||||
|
**Shedding** (optional, off by default): a domain `hot` for longer than its
|
||||||
|
shed delay has load moved off it using step 2's machinery. The lowest class goes
|
||||||
|
first, and replicas above their model's minimum go before any at it. A critical
|
||||||
|
minimum is never touched, and moves are rate limited to one replica per domain
|
||||||
|
per interval. The replica is placed elsewhere first where there is room, so
|
||||||
|
shedding moves work rather than dropping it. GPU power capping (`nvidia-smi
|
||||||
|
-pl`) is a gentler alternative worth measuring against shedding: less
|
||||||
|
throughput, no restart.
|
||||||
|
|
||||||
|
### Spreading
|
||||||
|
|
||||||
|
Replicas of one model are spread across racks, and across PDUs, the way they
|
||||||
|
are spread across nodes today. This is a preference, dropped when it is the only
|
||||||
|
thing standing in the way, as node spreading already is.
|
||||||
|
|
||||||
|
### Scoring
|
||||||
|
|
||||||
|
Placement today filters, then takes the first candidate in a fixed order
|
||||||
|
(fewest free GPUs for GPU profiles, fewest replicas for CPU ones, nodes holding
|
||||||
|
the weights first). Facility signals are better as a score than as more
|
||||||
|
filters. Candidates that pass the hard limits are ordered by:
|
||||||
|
|
||||||
|
1. condition: `ok` before `warm`;
|
||||||
|
2. power headroom left in the rack after this replica, more first, so load
|
||||||
|
levels across racks;
|
||||||
|
3. the existing order.
|
||||||
|
|
||||||
|
Once measurements include power, step 4's profile preference can use tokens per
|
||||||
|
watt instead of tokens per GPU where both are known.
|
||||||
|
|
||||||
|
## Where the data comes from
|
||||||
|
|
||||||
|
A `FacilitySource` interface, implemented several ways, chosen in
|
||||||
|
`server.yaml`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type FacilitySource interface {
|
||||||
|
// Domains is what the source knows: budgets, thresholds, members.
|
||||||
|
Domains(ctx) ([]Domain, error)
|
||||||
|
// Readings is the latest value of each signal, per domain.
|
||||||
|
Readings(ctx) ([]Reading, error)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Static file.** Domains, caps and thresholds, written by hand or generated
|
||||||
|
from a DCIM export. Enough for power budgets with no live data at all.
|
||||||
|
- **Prometheus.** DCIM, PDU (SNMP exporter) and BMC (Redfish exporter)
|
||||||
|
readings usually end up in Prometheus. The source runs configured instant
|
||||||
|
queries and maps their labels onto domains:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
facility:
|
||||||
|
source: prometheus
|
||||||
|
prometheus: http://monitoring:9090
|
||||||
|
signals:
|
||||||
|
inlet_temp_c:
|
||||||
|
query: max by (rack) (dcim_rack_inlet_temperature_celsius)
|
||||||
|
domain_label: rack
|
||||||
|
stale_after: 2m
|
||||||
|
power_watts:
|
||||||
|
query: sum by (pdu) (pdu_outlet_power_watts)
|
||||||
|
domain_label: pdu
|
||||||
|
cooling_alarm:
|
||||||
|
query: max by (row) (crac_alarm)
|
||||||
|
domain_label: row
|
||||||
|
```
|
||||||
|
- **Agents (built, step 0).** What each node reads about itself, every ten
|
||||||
|
seconds at most, sent with its heartbeat:
|
||||||
|
- **GPUs, from nvidia-smi:** temperature, memory (HBM) temperature where the
|
||||||
|
driver reports it, power draw, the enforced power limit, and what is
|
||||||
|
holding the clocks down now — `sw_power_cap`, `hw_slowdown`, `sw_thermal`,
|
||||||
|
`hw_thermal`, `power_brake`.
|
||||||
|
- **CPUs, from hwmon** (what lm_sensors reads, without it installed): the
|
||||||
|
package temperature, or the hottest core; and the board's inlet or ambient
|
||||||
|
sensor where it has one (`SYSTIN`, `Inlet`, `Ambient`).
|
||||||
|
- **CPU power, from RAPL:** the packages' energy counters over time,
|
||||||
|
subdomains not counted twice, counter wrap handled.
|
||||||
|
|
||||||
|
A machine without a sensor reports nothing for it, never a zero; a VM has
|
||||||
|
none of them. Published as `llmgw_gpu_temperature_celsius`,
|
||||||
|
`llmgw_gpu_memory_temperature_celsius`, `llmgw_gpu_power_watts`,
|
||||||
|
`llmgw_gpu_power_limit_watts`, `llmgw_gpu_throttled{reason}`,
|
||||||
|
`llmgw_node_cpu_temperature_celsius`, `llmgw_node_cpu_power_watts` and
|
||||||
|
`llmgw_node_inlet_temperature_celsius`, and on each node's page.
|
||||||
|
|
||||||
|
What they can and cannot say:
|
||||||
|
- **Throttling is the strongest signal there is.** A temperature is a proxy;
|
||||||
|
a card holding its clocks down is the fact that invalidates its measured
|
||||||
|
capacity, and it needs no threshold to be chosen.
|
||||||
|
- **Measured power** is GPU draw plus CPU packages; memory, disks, fans and
|
||||||
|
power-supply losses are not in it, so a node's draw is that plus an
|
||||||
|
estimated remainder. A rack's is its llmgw nodes' plus a declared figure
|
||||||
|
for everything else in it: switches and storage report nothing here.
|
||||||
|
- **CPU and GPU temperatures follow the load, not the room.** A hot rack is
|
||||||
|
inferred from many nodes running hot at comparable load, or from
|
||||||
|
throttling, not from one node's CPU. Only a board inlet sensor reads the
|
||||||
|
air, and most boards have none.
|
||||||
|
- **No budget comes from a sensor.** A reading is a draw; the cap it is
|
||||||
|
measured against still comes from the facility.
|
||||||
|
- **Simulated.** Readings from a script, for tests and for rehearsing a
|
||||||
|
thermal event on a cluster without facility data.
|
||||||
|
|
||||||
|
## Safety
|
||||||
|
|
||||||
|
- **Missing data never blocks.** A source that is down, a reading that is stale,
|
||||||
|
a node without location labels: placement behaves as it does today, and says
|
||||||
|
the data is missing (a metric, a line on the cluster page). Facility
|
||||||
|
awareness can make placement more careful, never impossible.
|
||||||
|
- **Hysteresis and hold times** on every condition, so a noisy sensor does not
|
||||||
|
flap a rack between allowed and forbidden.
|
||||||
|
- **Observe-only first.** A mode where every decision is computed, logged and
|
||||||
|
shown ("would have refused: rack r12-07 hot") but nothing is enforced, run
|
||||||
|
for long enough to trust the thresholds before turning enforcement on.
|
||||||
|
- **Manual override.** `llmgw ctl facility hold RACK` and `release RACK`, for the
|
||||||
|
facility team to mark a domain unavailable, or to clear a condition they know
|
||||||
|
to be a sensor fault.
|
||||||
|
- **Shedding is opt-in**, per domain or cluster-wide, and rate limited.
|
||||||
|
|
||||||
|
## What people see
|
||||||
|
|
||||||
|
- `explain-placement` and headroom name the domain: *rack r12-07 is hot (inlet
|
||||||
|
31°C for 6 min)*, *pdu r12-07-a: 7.1 of 7.4 kW*.
|
||||||
|
- `llmgw ctl facility` lists domains with budget, reserved, measured and
|
||||||
|
condition. The cluster page gets a rack summary beside the nodes table.
|
||||||
|
- Metrics: `llmgw_domain_power_budget_watts`, `_reserved_watts`,
|
||||||
|
`_measured_watts`, `llmgw_domain_condition{condition}`,
|
||||||
|
`llmgw_domain_reading_age_seconds`; alerts for a domain near its cap, a
|
||||||
|
domain hot, a domain over its cap, and facility data gone stale.
|
||||||
|
|
||||||
|
## Rollout
|
||||||
|
|
||||||
|
Each step ships on its own and is useful without the next:
|
||||||
|
|
||||||
|
0. **Node sensors, observe-only (built).** GPU heat, power and throttling,
|
||||||
|
CPU heat and power, inlet air where the board has a sensor; published and
|
||||||
|
shown, no decision changed.
|
||||||
|
1. **Location and spreading.** Well-known labels, domains from labels, replicas
|
||||||
|
spread across racks and PDUs, locations on the node views. No facility data
|
||||||
|
needed.
|
||||||
|
2. **Power budgets from a static file.** Caps and margins, estimated draw per
|
||||||
|
replica (declared or derived), hard limit in placement and headroom.
|
||||||
|
3. **Readings, observe-only.** The Prometheus source beside the agents',
|
||||||
|
conditions with hysteresis (throttling first among them), measured power
|
||||||
|
folded into headroom; decisions shown, not enforced.
|
||||||
|
4. **Thermal enforcement.** `warm` scored down, `hot` refused, manual hold and
|
||||||
|
release.
|
||||||
|
5. **Shedding and tokens per watt.** Opt-in shedding through step 2's
|
||||||
|
preemption; measured power in profile preference.
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
- The simulated source drives unit tests of every condition transition, the
|
||||||
|
hysteresis, staleness, and the placement outcomes, with mutation checks as
|
||||||
|
for steps 1–4.
|
||||||
|
- Property tests: whatever the readings, placement never exceeds a declared
|
||||||
|
cap with fresh data, and never refuses everything because data is missing.
|
||||||
|
- On the PoC cluster, with no facility data: step 1 with labelled VMs, step 2
|
||||||
|
with a static file of made-up caps, steps 3–4 with the simulated source
|
||||||
|
replaying a thermal event.
|
||||||
|
- Before enforcement anywhere real: a period in observe-only mode against the
|
||||||
|
site's actual data.
|
||||||
|
|
||||||
|
## Open questions
|
||||||
|
|
||||||
|
- **What the sites have.** Which DCIM, which PDUs, which BMCs, and whether any
|
||||||
|
of it already reaches a Prometheus. The source design assumes Prometheus is
|
||||||
|
the common ground; a site without it needs another source.
|
||||||
|
- **Who owns the numbers.** Caps, deratings and thresholds belong to the
|
||||||
|
facility team. Whether they edit llmgw's file, or llmgw reads theirs.
|
||||||
|
- **Phase balance.** Three-phase PDUs have per-phase limits; whether placement
|
||||||
|
should see phases or only the PDU total.
|
||||||
|
- **Cooling granularity.** Cooling is often per row or per aisle, not per rack;
|
||||||
|
the domain tree allows either, but the defaults should match what sites
|
||||||
|
actually measure.
|
||||||
|
- **Power capping instead of shedding.** Whether to lower GPU power limits in a
|
||||||
|
warm rack rather than move replicas, and how that interacts with measured
|
||||||
|
capacity from step 4, which would no longer hold at the lower limit.
|
||||||
+194
-46
@@ -671,6 +671,74 @@ llmgw ctl explain-placement qwen2.5-coder-3b-instruct
|
|||||||
which names every candidate and why each was rejected, rather than leaving you
|
which names every candidate and why each was rejected, rather than leaving you
|
||||||
to infer it.
|
to infer it.
|
||||||
|
|
||||||
|
### LoRA adapters
|
||||||
|
|
||||||
|
A llama.cpp model can carry LoRA adapters, each asked for by its own name as
|
||||||
|
though it were a model:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- name: qwen2.5-coder-7b-instruct
|
||||||
|
source: http://mirror:8081/qwen2.5-coder-7b-instruct/5d567d26/model.gguf
|
||||||
|
adapters:
|
||||||
|
- name: coder-sql
|
||||||
|
source: http://mirror:8081/coder-sql/v1/adapter.gguf
|
||||||
|
scale: 1.0 # optional; 1 when left out
|
||||||
|
profiles: [...]
|
||||||
|
```
|
||||||
|
|
||||||
|
Every replica of the model fetches every adapter and loads it at start, applied
|
||||||
|
to nothing. A request for `coder-sql` goes to the model's replicas with the
|
||||||
|
adapter applied to it alone; a request for the model gets the model. Adapter
|
||||||
|
names share the namespace of models and aliases, appear in `/v1/models`, and
|
||||||
|
follow key allowlists on their own: a key allowed `coder-sql` reaches that
|
||||||
|
adapter and not the model under it, while a key allowed the model reaches its
|
||||||
|
adapters too. Adding or removing an adapter is a change to the model; running
|
||||||
|
replicas load the new set when restarted (`llmgw ctl restart --model M`).
|
||||||
|
Besides the mirror page's model card, adapters are attached from the command
|
||||||
|
line, which reads the file first and refuses one that is not an adapter for
|
||||||
|
this model's architecture:
|
||||||
|
|
||||||
|
llmgw ctl model adapters coder --add coder-sql=http://mirror:8081/sql-lora/v1/sql.gguf@0.8
|
||||||
|
llmgw ctl model adapters coder --remove coder-sql
|
||||||
|
|
||||||
|
The adapter file must be a GGUF LoRA made for this model — llama.cpp's
|
||||||
|
`convert_lora_to_gguf.py` makes one from a PEFT adapter — and not a model with
|
||||||
|
the adapter already merged in, which is what most "LoRA GGUF" repositories hold.
|
||||||
|
|
||||||
|
### Models served outside the cluster
|
||||||
|
|
||||||
|
A model can be an OpenAI-compatible API elsewhere — a hosted provider, or
|
||||||
|
another gateway — rather than replicas here. Nothing is placed for it; the
|
||||||
|
router sends its requests on, with the provider's key and the provider's name
|
||||||
|
for the model:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- name: cloud-mini
|
||||||
|
source: external
|
||||||
|
external:
|
||||||
|
url: https://api.openai.com # without /v1
|
||||||
|
model: gpt-4o-mini # what the provider calls it
|
||||||
|
api_key_file: /etc/llmgw/secrets/openai.key
|
||||||
|
```
|
||||||
|
|
||||||
|
Named in another model's `fallbacks`, it is used only when the cluster has no
|
||||||
|
capacity left for that model:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- name: qwen2.5-coder-7b-instruct
|
||||||
|
fallbacks: [cloud-mini]
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
The key is read from its file on the control plane, never stored in the
|
||||||
|
catalog, and a key rotated in the file is used within seconds. A model whose key
|
||||||
|
file cannot be read is not offered. Everything else applies as to any model:
|
||||||
|
prices and budgets, rate limits, usage records, and key allowlists.
|
||||||
|
|
||||||
|
**Its prompts leave the cluster.** Decide who may send them: a key or project
|
||||||
|
whose allowlist leaves the external model out cannot reach it, as a fallback or
|
||||||
|
by name. The cluster page and `ctl models` mark it `external`.
|
||||||
|
|
||||||
### How many replicas, and how much each one carries
|
### How many replicas, and how much each one carries
|
||||||
|
|
||||||
Replica count is changed while the cluster runs, without editing the
|
Replica count is changed while the cluster runs, without editing the
|
||||||
@@ -909,88 +977,160 @@ refuses every request.
|
|||||||
|
|
||||||
## Upgrading
|
## Upgrading
|
||||||
|
|
||||||
One binary on every machine, so an upgrade is a copy and a restart. The part
|
On the control plane:
|
||||||
worth care is the copy, not the order.
|
|
||||||
|
```
|
||||||
|
sudo llmgw ctl upgrade v0.10.4
|
||||||
|
```
|
||||||
|
|
||||||
|
It upgrades this machine first — fetching the release for its platform,
|
||||||
|
checking it against the release's checksums, keeping the old binary as
|
||||||
|
`/usr/local/bin/llmgw.bak-pre-<version>`, restarting `llmgw-server` and waiting
|
||||||
|
for every node to reconnect — and then each node in turn. A node's agent is told
|
||||||
|
the version over the stream it already holds, fetches and checks the release
|
||||||
|
itself, swaps it in and restarts; no ssh into the nodes is needed. Release
|
||||||
|
candidates (`v0.10.4-rc.2`) work the same way.
|
||||||
|
|
||||||
|
A node counts as done when it is back on the new version, ready, with as many
|
||||||
|
ready replicas as before. One that is not within `--timeout` (three minutes)
|
||||||
|
stops the upgrade there, so a bad release reaches one node rather than all of
|
||||||
|
them; `llmgw ctl logs <node>` says why, and the node keeps its previous binary
|
||||||
|
beside the new one. Nodes already on the version are skipped, so running the
|
||||||
|
command again resumes where it stopped. `--nodes a,b` limits it to some nodes,
|
||||||
|
and `--skip-control-plane` leaves this machine alone.
|
||||||
|
|
||||||
**Nothing stops serving.** Engines are started with `KillMode=process`, so they
|
**Nothing stops serving.** Engines are started with `KillMode=process`, so they
|
||||||
keep running when the agent restarts, and the agent reports them back on its
|
keep running when the agent restarts, and the agent reports them back on its
|
||||||
next connection. The control plane holds no replica state across its own
|
next connection. The control plane holds no replica state across its own
|
||||||
restart either — it rebuilds the fleet from what the agents report. A rolling
|
restart either — it rebuilds the fleet from what the agents report.
|
||||||
upgrade of the whole cluster interrupts no request that is not in flight at the
|
|
||||||
moment of the restart.
|
|
||||||
|
|
||||||
**A mixed fleet is fine while you roll.** The agent protocol only ever gains
|
**A mixed fleet is fine while you roll.** The agent protocol only ever gains
|
||||||
fields, and both sides treat an absent one as unknown rather than as a default.
|
fields, and both sides treat an absent one as unknown rather than as a default.
|
||||||
Order does not matter; finish the roll the same day anyway, because
|
Agents older than v0.10.4 cannot upgrade themselves: move those by hand once,
|
||||||
`ctl nodes` reports each agent's version and a fleet that disagrees with itself
|
installing the release binary and restarting `llmgw-agent`, and every upgrade
|
||||||
is one more thing to hold in your head.
|
after that is the command above.
|
||||||
|
|
||||||
|
**The mirror is upgraded on its own host**, since it is not an agent and its
|
||||||
|
service cannot replace its own binary:
|
||||||
|
|
||||||
```
|
```
|
||||||
VERSION=v0.9.0
|
sudo llmgw upgrade v0.10.5 --service llmgw-mirror
|
||||||
# copy to every machine first, install nowhere yet
|
|
||||||
for h in 10.0.0.140 10.0.0.141 10.0.0.142 10.0.0.146; do
|
|
||||||
ssh $h 'cat > /tmp/llmgw-new; stat -c %s /tmp/llmgw-new' < llmgw-$VERSION-linux-amd64
|
|
||||||
done
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**Check every size before installing anything.** This is not defensive writing:
|
The same checks — the release's checksums, the old binary kept beside the new —
|
||||||
on a host whose login shell prints anything, `scp` fails outright, and the
|
then a restart of the unit, waiting for it to stay up. A mirror older than
|
||||||
obvious workaround — `ssh -n host 'cat > file'` — points stdin at `/dev/null`
|
v0.10.5 does not have the command: install the release binary by hand once.
|
||||||
and writes a *zero-byte file* while reporting success. Installing that on a
|
|
||||||
fleet takes the fleet down. Compare the number on each machine with the local
|
|
||||||
one, then:
|
|
||||||
|
|
||||||
```
|
|
||||||
sudo install -m0755 /tmp/llmgw-new /usr/local/bin/llmgw
|
|
||||||
sudo systemctl restart llmgw-agent # or llmgw-server, or llmgw-mirror
|
|
||||||
llmgw --version
|
|
||||||
```
|
|
||||||
|
|
||||||
Then confirm what you have, rather than assuming:
|
Then confirm what you have, rather than assuming:
|
||||||
|
|
||||||
```
|
```
|
||||||
llmgw ctl nodes # every agent back, LAST SEEN in seconds
|
llmgw ctl nodes # every agent on the new version, LAST SEEN in seconds
|
||||||
llmgw ctl replicas # same replicas, adopted rather than replaced
|
llmgw ctl replicas # same replicas, adopted rather than replaced
|
||||||
```
|
```
|
||||||
|
|
||||||
A replica may come back marked **`ready (stale profile)`**. That means it is
|
A replica may come back marked **`ready (stale profile)`**. That means it is
|
||||||
running a profile that has since been edited — the reconciler leaves a serving
|
running a profile that has since been edited — the reconciler leaves a serving
|
||||||
replica alone on purpose, because churning one costs more than the difference.
|
replica alone on purpose, because churning one costs more than the difference.
|
||||||
Restart it when you want the new profile to take effect; the flag is there so
|
`llmgw ctl restart <replica>` replaces it with one on the new profile, starting
|
||||||
the choice is yours rather than silent.
|
the new one before stopping the old.
|
||||||
|
|
||||||
|
### Signed releases
|
||||||
|
|
||||||
|
Every install above checks a release against its published checksums, which
|
||||||
|
proves the download is what the server holds — not that the server holds what
|
||||||
|
was built. Signing closes that: CI signs the checksums, and a binary that
|
||||||
|
trusts a key installs only releases signed with it. Off until a key exists;
|
||||||
|
turning it on, once:
|
||||||
|
|
||||||
|
```
|
||||||
|
llmgw release keygen
|
||||||
|
```
|
||||||
|
|
||||||
|
1. Put the **private** key in the repository's Actions secret
|
||||||
|
`RELEASE_SIGNING_KEY`. From the next tag, releases carry
|
||||||
|
`checksums.txt.sig`.
|
||||||
|
2. Put the **public** key in `internal/release/trusted.pub` and release. Every
|
||||||
|
binary built from then on carries it and refuses releases that are not
|
||||||
|
signed with it — `ctl upgrade`, agents upgrading themselves, and
|
||||||
|
`llmgw upgrade` alike.
|
||||||
|
|
||||||
|
In that order: a build that trusts a key refuses unsigned releases, so the
|
||||||
|
secret has to be signing before the key is trusted. The first trusting release
|
||||||
|
is itself installed by a binary that trusts nothing; every one after is checked
|
||||||
|
by the one before. `LLMGW_RELEASE_KEY` overrides the built-in key, for rotating
|
||||||
|
it. The onboarding script fetches with `curl` and checks checksums only; a node
|
||||||
|
is checked from its first upgrade on.
|
||||||
|
|
||||||
## Backing up
|
## Backing up
|
||||||
|
|
||||||
Three things, in descending order of how badly you want them.
|
`llmgw server backup` writes one archive holding everything a control plane
|
||||||
|
cannot be rebuilt without:
|
||||||
|
|
||||||
**The certificate authority.** `ca-key.pem`, `/etc/llmgw/pki` unless you kept it elsewhere (step 2).
|
- **the store** — users, API keys, the tenancy tree, limits and budgets, usage
|
||||||
Losing it does not stop the cluster, and it means every node identity has to be
|
history, models added at runtime, measured capacity, and the secret every
|
||||||
reissued the next time you touch one. Nothing in llmgw can reconstruct it.
|
engine key is derived from (`pg_dump` in its custom format for PostgreSQL, a
|
||||||
|
consistent `VACUUM INTO` snapshot for SQLite; both are safe against a running
|
||||||
|
gateway);
|
||||||
|
- **`/etc/llmgw`** — the configuration, the store's address, and the
|
||||||
|
certificate authority's key (`pki/`). Nothing in llmgw can reconstruct that
|
||||||
|
key; losing it means reissuing every node's identity.
|
||||||
|
|
||||||
**The store.** Users, API keys, organisations, teams, projects, every limit and
|
Archives go to `/var/backups/llmgw-server`, readable by root only because of the CA
|
||||||
budget, all usage history, models added at runtime, and measured capacity.
|
key, and the newest 14 are kept. Run it nightly with the timer:
|
||||||
|
|
||||||
```
|
```
|
||||||
# SQLite
|
sudo install -m 0644 deploy/systemd/llmgw-backup.service deploy/systemd/llmgw-backup.timer /etc/systemd/system/
|
||||||
sqlite3 /var/lib/llmgw/llmgw.db ".backup '/backup/llmgw-$(date +%F).db'"
|
sudo systemctl daemon-reload
|
||||||
|
sudo systemctl enable --now llmgw-backup.timer
|
||||||
# PostgreSQL
|
sudo systemctl start llmgw-backup.service # one now, to see it work
|
||||||
pg_dump --format=custom llmgw > /backup/llmgw-$(date +%F).dump
|
ls -l /var/backups/llmgw-server
|
||||||
```
|
```
|
||||||
|
|
||||||
`.backup` and `pg_dump` are both safe against a running gateway. Copying the
|
The control plane publishes when the newest archive was written as
|
||||||
SQLite file with `cp` while it is being written is not.
|
`llmgw_backup_last_success_timestamp_seconds`, and **BackupTooOld** fires when
|
||||||
|
it is more than 26 hours old — including when there has never been one.
|
||||||
|
|
||||||
|
**Copy the archives off the machine.** They sit on the disk they back up and do
|
||||||
|
not survive losing it. Anything that copies files will do — rsync from the
|
||||||
|
mirror, a NAS mount, an offsite bucket — as long as it keeps the archives
|
||||||
|
readable by root only.
|
||||||
|
|
||||||
**Keys are stored hashed and cannot be recovered from a backup or from
|
**Keys are stored hashed and cannot be recovered from a backup or from
|
||||||
anywhere else.** Restoring an old store restores which keys exist and who owns
|
anywhere else.** Restoring an old store restores which keys exist and who owns
|
||||||
them, not their secrets. A key created after the backup is gone and has to be
|
them, not their secrets. A key created after the backup is gone and has to be
|
||||||
reissued — which is why the store is worth backing up on a schedule rather than
|
reissued.
|
||||||
before upgrades.
|
|
||||||
|
### Restoring
|
||||||
|
|
||||||
|
On a machine with llmgw installed (the same version or newer) and, for
|
||||||
|
PostgreSQL, an empty database and role as `server init --postgres` would create
|
||||||
|
them:
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo systemctl stop llmgw-server
|
||||||
|
mkdir /tmp/restore && sudo tar -xzf /var/backups/llmgw-server/llmgw-backup-<host>-<time>.tar.gz -C /tmp/restore
|
||||||
|
cat /tmp/restore/MANIFEST # version and time it was taken
|
||||||
|
|
||||||
|
# /etc/llmgw: configuration, secrets, certificates and the CA
|
||||||
|
sudo cp -a /tmp/restore/etc/llmgw/. /etc/llmgw/
|
||||||
|
|
||||||
|
# the store — PostgreSQL
|
||||||
|
sudo -u postgres pg_restore --clean --if-exists --no-owner --role=llmgw -d llmgw /tmp/restore/store.pgdump
|
||||||
|
# the store — SQLite
|
||||||
|
sudo install -o llmgw -g llmgw -m 0600 /tmp/restore/store.sqlite /var/lib/llmgw/llmgw.db
|
||||||
|
|
||||||
|
sudo systemctl start llmgw-server
|
||||||
|
sudo rm -rf /tmp/restore
|
||||||
|
```
|
||||||
|
|
||||||
|
Running engines keep serving throughout: agents reconnect to the restored
|
||||||
|
control plane and their replicas are adopted, and because the engine-key secret
|
||||||
|
came back with the store, the router can still reach them.
|
||||||
|
|
||||||
**The mirror's directory** is the one thing you can rebuild, by downloading it
|
**The mirror's directory** is the one thing you can rebuild, by downloading it
|
||||||
all again. Worth remembering that a pinned version may no longer be upstream to
|
all again. A pinned version may no longer be upstream to download: that is
|
||||||
download: that is exactly what pinning it protected you from, and a backup is
|
exactly what pinning it protected you from, and a copy of the directory is what
|
||||||
what makes the protection outlive the disk.
|
makes the protection outlive the disk.
|
||||||
|
|
||||||
## Reclaiming disk
|
## Reclaiming disk
|
||||||
|
|
||||||
@@ -1089,3 +1229,11 @@ requests. On a slow model, set `timeouts.generation` higher.
|
|||||||
has them. Single-node placement on real GPUs is the well-trodden path; if you
|
has them. Single-node placement on real GPUs is the well-trodden path; if you
|
||||||
are the first to run a TP group across real cards, run `llmgw doctor --json`
|
are the first to run a TP group across real cards, run `llmgw doctor --json`
|
||||||
on each node first and keep the reports.
|
on each node first and keep the reports.
|
||||||
|
- **Two engine setups are weaker than the rest.** An engine's key is passed
|
||||||
|
in its environment, where only root and its own user can read it, except
|
||||||
|
SGLang's, which takes it only as a flag: other users on the node can see it
|
||||||
|
in the process list. Mount `/proc` with `hidepid=2` on nodes where that
|
||||||
|
matters. vLLM's `launch_mode: ray` starts a Ray head without
|
||||||
|
authentication, so anyone who can reach its port can run code on the
|
||||||
|
node. The dashboard is off, but firewall the port to the other nodes in the
|
||||||
|
replica, or use the default `mp` launch mode, which needs no Ray at all.
|
||||||
@@ -15,12 +15,16 @@ rather than repeated.
|
|||||||
|
|
||||||
| Key | Type | In |
|
| Key | Type | In |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
|
| `name` | `string` | [Adapter](#adapter) |
|
||||||
|
| `source` | `string` | [Adapter](#adapter) |
|
||||||
|
| `scale` | `float64` | [Adapter](#adapter) |
|
||||||
| `enabled` | `bool` | [Admission](#admission) |
|
| `enabled` | `bool` | [Admission](#admission) |
|
||||||
| `bands` | `[]Band` | [Admission](#admission) |
|
| `bands` | `[]Band` | [Admission](#admission) |
|
||||||
| `default_band` | `string` | [Admission](#admission) |
|
| `default_band` | `string` | [Admission](#admission) |
|
||||||
| `band_header` | `string` | [Admission](#admission) |
|
| `band_header` | `string` | [Admission](#admission) |
|
||||||
| `queue_ttl` | `time.Duration` | [Admission](#admission) |
|
| `queue_ttl` | `time.Duration` | [Admission](#admission) |
|
||||||
| `queue_capacity_per_band` | `int` | [Admission](#admission) |
|
| `queue_capacity_per_band` | `int` | [Admission](#admission) |
|
||||||
|
| `queue_capacity_total` | `int` | [Admission](#admission) |
|
||||||
| `interval` | `time.Duration` | [Admission](#admission) |
|
| `interval` | `time.Duration` | [Admission](#admission) |
|
||||||
| `enabled` | `bool` | [Auth](#auth) |
|
| `enabled` | `bool` | [Auth](#auth) |
|
||||||
| `store` | `string` | [Auth](#auth) |
|
| `store` | `string` | [Auth](#auth) |
|
||||||
@@ -71,15 +75,20 @@ rather than repeated.
|
|||||||
| `lease_ttl` | `time.Duration` | [Control](#control) |
|
| `lease_ttl` | `time.Duration` | [Control](#control) |
|
||||||
| `id` | `string` | [Endpoint](#endpoint) |
|
| `id` | `string` | [Endpoint](#endpoint) |
|
||||||
| `url` | `string` | [Endpoint](#endpoint) |
|
| `url` | `string` | [Endpoint](#endpoint) |
|
||||||
|
| `api_key` | `string` | [Endpoint](#endpoint) |
|
||||||
| `models` | `[]string` | [Endpoint](#endpoint) |
|
| `models` | `[]string` | [Endpoint](#endpoint) |
|
||||||
| `engine` | `string` | [Endpoint](#endpoint) |
|
| `engine` | `string` | [Endpoint](#endpoint) |
|
||||||
| `pool` | `string` | [Endpoint](#endpoint) |
|
| `pool` | `string` | [Endpoint](#endpoint) |
|
||||||
| `weight` | `float64` | [Endpoint](#endpoint) |
|
| `weight` | `float64` | [Endpoint](#endpoint) |
|
||||||
| `priority` | `int` | [Endpoint](#endpoint) |
|
| `priority` | `int` | [Endpoint](#endpoint) |
|
||||||
|
| `url` | `string` | [External](#external) |
|
||||||
|
| `model` | `string` | [External](#external) |
|
||||||
|
| `api_key_file` | `string` | [External](#external) |
|
||||||
| `contact` | `string` | [Landing](#landing) |
|
| `contact` | `string` | [Landing](#landing) |
|
||||||
| `notice` | `string` | [Landing](#landing) |
|
| `notice` | `string` | [Landing](#landing) |
|
||||||
| `bin` | `string` | [LlamaCppConfig](#llamacppconfig) |
|
| `bin` | `string` | [LlamaCppConfig](#llamacppconfig) |
|
||||||
| `parallel` | `int` | [LlamaCppConfig](#llamacppconfig) |
|
| `parallel` | `int` | [LlamaCppConfig](#llamacppconfig) |
|
||||||
|
| `threads` | `int` | [LlamaCppConfig](#llamacppconfig) |
|
||||||
| `api` | `string` | [Mirror](#mirror) |
|
| `api` | `string` | [Mirror](#mirror) |
|
||||||
| `tls` | `tlsconf.Config` | [Mirror](#mirror) |
|
| `tls` | `tlsconf.Config` | [Mirror](#mirror) |
|
||||||
| `registry` | `string` | [Mirror](#mirror) |
|
| `registry` | `string` | [Mirror](#mirror) |
|
||||||
@@ -90,6 +99,10 @@ rather than repeated.
|
|||||||
| `source` | `string` | [Model](#model) |
|
| `source` | `string` | [Model](#model) |
|
||||||
| `profiles` | `[]Profile` | [Model](#model) |
|
| `profiles` | `[]Profile` | [Model](#model) |
|
||||||
| `scaling` | `Scaling` | [Model](#model) |
|
| `scaling` | `Scaling` | [Model](#model) |
|
||||||
|
| `class` | `string` | [Model](#model) |
|
||||||
|
| `owner` | `string` | [Model](#model) |
|
||||||
|
| `external` | `*External` | [Model](#model) |
|
||||||
|
| `adapters` | `[]Adapter` | [Model](#model) |
|
||||||
| `fallbacks` | `[]string` | [Model](#model) |
|
| `fallbacks` | `[]string` | [Model](#model) |
|
||||||
| `routing_profile` | `string` | [Model](#model) |
|
| `routing_profile` | `string` | [Model](#model) |
|
||||||
| `slo` | `SLO` | [Model](#model) |
|
| `slo` | `SLO` | [Model](#model) |
|
||||||
@@ -127,6 +140,7 @@ rather than repeated.
|
|||||||
| `args` | `[]string` | [Profile](#profile) |
|
| `args` | `[]string` | [Profile](#profile) |
|
||||||
| `env` | `map[string]string` | [Profile](#profile) |
|
| `env` | `map[string]string` | [Profile](#profile) |
|
||||||
| `min_memory_gb` | `int` | [Profile](#profile) |
|
| `min_memory_gb` | `int` | [Profile](#profile) |
|
||||||
|
| `cpu_cores` | `int` | [Profile](#profile) |
|
||||||
| `min_disk_gb` | `int` | [Profile](#profile) |
|
| `min_disk_gb` | `int` | [Profile](#profile) |
|
||||||
| `node_selector` | `map[string]string` | [Profile](#profile) |
|
| `node_selector` | `map[string]string` | [Profile](#profile) |
|
||||||
| `weight` | `float64` | [Profile](#profile) |
|
| `weight` | `float64` | [Profile](#profile) |
|
||||||
@@ -166,8 +180,11 @@ rather than repeated.
|
|||||||
| `landing` | `Landing` | [Server](#server) |
|
| `landing` | `Landing` | [Server](#server) |
|
||||||
| `agent_tls` | `tlsconf.Config` | [Server](#server) |
|
| `agent_tls` | `tlsconf.Config` | [Server](#server) |
|
||||||
| `monitoring` | `Monitoring` | [Server](#server) |
|
| `monitoring` | `Monitoring` | [Server](#server) |
|
||||||
|
| `tracing` | `Tracing` | [Server](#server) |
|
||||||
| `mirror` | `Mirror` | [Server](#server) |
|
| `mirror` | `Mirror` | [Server](#server) |
|
||||||
| `engines` | `map[string]string` | [Server](#server) |
|
| `engines` | `map[string]string` | [Server](#server) |
|
||||||
|
| `backup_dir` | `string` | [Server](#server) |
|
||||||
|
| `releases` | `string` | [Server](#server) |
|
||||||
| `router` | `Router` | [Server](#server) |
|
| `router` | `Router` | [Server](#server) |
|
||||||
| `control` | `Control` | [Server](#server) |
|
| `control` | `Control` | [Server](#server) |
|
||||||
| `models` | `[]catalog.Model` | [Server](#server) |
|
| `models` | `[]catalog.Model` | [Server](#server) |
|
||||||
@@ -181,10 +198,33 @@ rather than repeated.
|
|||||||
| `first_token` | `time.Duration` | [Timeouts](#timeouts) |
|
| `first_token` | `time.Duration` | [Timeouts](#timeouts) |
|
||||||
| `idle` | `time.Duration` | [Timeouts](#timeouts) |
|
| `idle` | `time.Duration` | [Timeouts](#timeouts) |
|
||||||
| `total` | `time.Duration` | [Timeouts](#timeouts) |
|
| `total` | `time.Duration` | [Timeouts](#timeouts) |
|
||||||
|
| `endpoint` | `string` | [Tracing](#tracing) |
|
||||||
|
| `sample_ratio` | `float64` | [Tracing](#tracing) |
|
||||||
| `bin` | `string` | [VLLMConfig](#vllmconfig) |
|
| `bin` | `string` | [VLLMConfig](#vllmconfig) |
|
||||||
| `launch_mode` | `string` | [VLLMConfig](#vllmconfig) |
|
| `launch_mode` | `string` | [VLLMConfig](#vllmconfig) |
|
||||||
| `ray_bin` | `string` | [VLLMConfig](#vllmconfig) |
|
| `ray_bin` | `string` | [VLLMConfig](#vllmconfig) |
|
||||||
|
|
||||||
|
## Adapter
|
||||||
|
|
||||||
|
### `name`
|
||||||
|
|
||||||
|
`string`
|
||||||
|
|
||||||
|
Name is what clients ask for, like a model name.
|
||||||
|
|
||||||
|
### `source`
|
||||||
|
|
||||||
|
`string`
|
||||||
|
|
||||||
|
Source is the adapter file (GGUF for llama.cpp), fetched by each node
|
||||||
|
as the model's weights are.
|
||||||
|
|
||||||
|
### `scale`
|
||||||
|
|
||||||
|
`float64`
|
||||||
|
|
||||||
|
Scale is how strongly it is applied; zero takes 1.
|
||||||
|
|
||||||
## Admission
|
## Admission
|
||||||
|
|
||||||
### `enabled`
|
### `enabled`
|
||||||
@@ -218,6 +258,15 @@ should not be able to promote themselves.
|
|||||||
|
|
||||||
`int`
|
`int`
|
||||||
|
|
||||||
|
### `queue_capacity_total`
|
||||||
|
|
||||||
|
`int`
|
||||||
|
|
||||||
|
QueueCapacityTotal bounds everything queued across bands. Reached, a
|
||||||
|
request is not refused while a lower band has any queued: the newest of
|
||||||
|
the lowest band is turned away instead, with a 503 and Retry-After.
|
||||||
|
Zero takes half of all the bands' capacities together.
|
||||||
|
|
||||||
### `interval`
|
### `interval`
|
||||||
|
|
||||||
`time.Duration`
|
`time.Duration`
|
||||||
@@ -566,6 +615,13 @@ reading the alert. Zero takes the default.
|
|||||||
|
|
||||||
`string`
|
`string`
|
||||||
|
|
||||||
|
### `api_key`
|
||||||
|
|
||||||
|
`string`
|
||||||
|
|
||||||
|
APIKey is sent to the engine as a bearer token, for an engine started
|
||||||
|
with one.
|
||||||
|
|
||||||
### `models`
|
### `models`
|
||||||
|
|
||||||
`[]string`
|
`[]string`
|
||||||
@@ -589,6 +645,28 @@ called. Empty means vLLM's names.
|
|||||||
|
|
||||||
`int`
|
`int`
|
||||||
|
|
||||||
|
## External
|
||||||
|
|
||||||
|
### `url`
|
||||||
|
|
||||||
|
`string`
|
||||||
|
|
||||||
|
URL is the provider's base address, without /v1: https://api.openai.com.
|
||||||
|
|
||||||
|
### `model`
|
||||||
|
|
||||||
|
`string`
|
||||||
|
|
||||||
|
Model is what the provider calls it; requests are rewritten to it.
|
||||||
|
|
||||||
|
### `api_key_file`
|
||||||
|
|
||||||
|
`string`
|
||||||
|
|
||||||
|
APIKeyFile holds the provider's key, on the control plane. A file
|
||||||
|
rather than the key itself, so the key is in neither the catalog, the
|
||||||
|
store nor the audit log; empty for a provider that asks for none.
|
||||||
|
|
||||||
## Landing
|
## Landing
|
||||||
|
|
||||||
### `contact`
|
### `contact`
|
||||||
@@ -624,6 +702,20 @@ Parallel is how many requests a server handles at once, when a profile
|
|||||||
does not say. llama.cpp defaults to one, which would make every second
|
does not say. llama.cpp defaults to one, which would make every second
|
||||||
request wait behind the first no matter what the router decided.
|
request wait behind the first no matter what the router decided.
|
||||||
|
|
||||||
|
### `threads`
|
||||||
|
|
||||||
|
`int`
|
||||||
|
|
||||||
|
Threads is how many CPU threads a shard with no GPU computes on. Zero
|
||||||
|
takes half this machine's CPUs.
|
||||||
|
|
||||||
|
Not all of them, which is llama.cpp's own default: two busy servers
|
||||||
|
that each spin on every CPU fight for them, and on an 8-vCPU node
|
||||||
|
that made 7B generation fall from 10 tokens a second to under 1 each.
|
||||||
|
At half, one server alone is about a third slower and two together
|
||||||
|
keep 5.5 each. A node that only ever runs one CPU replica does better
|
||||||
|
with all of them; say so here.
|
||||||
|
|
||||||
## Mirror
|
## Mirror
|
||||||
|
|
||||||
### `api`
|
### `api`
|
||||||
@@ -707,6 +799,45 @@ Source is the weights path or a HuggingFace id.
|
|||||||
|
|
||||||
Takes a [Scaling](#scaling) block.
|
Takes a [Scaling](#scaling) block.
|
||||||
|
|
||||||
|
### `class`
|
||||||
|
|
||||||
|
`string`
|
||||||
|
|
||||||
|
Class is how much this model's capacity is worth against another's,
|
||||||
|
when there is not enough for both: critical, standard or opportunistic.
|
||||||
|
Empty is standard, or opportunistic for a model loaded on demand. See
|
||||||
|
EffectiveClass.
|
||||||
|
|
||||||
|
### `owner`
|
||||||
|
|
||||||
|
`string`
|
||||||
|
|
||||||
|
Owner is the part of the tenancy tree whose capacity quota this model
|
||||||
|
counts against, written org:ID, team:ID or project:ID. Empty is shared:
|
||||||
|
held by the cluster, counted against nobody.
|
||||||
|
|
||||||
|
### `external`
|
||||||
|
|
||||||
|
`*External`
|
||||||
|
|
||||||
|
External is a model served by someone else — an OpenAI-compatible API
|
||||||
|
outside the cluster — rather than by replicas placed here. Such a model
|
||||||
|
has no profiles and nothing is placed for it; the router sends its
|
||||||
|
requests to the provider. Most useful named in another model's
|
||||||
|
fallbacks: used only when the cluster has no capacity left. Its
|
||||||
|
prompts leave the cluster, so which keys may name it is worth an
|
||||||
|
allowlist.
|
||||||
|
|
||||||
|
Takes a [External](#external) block.
|
||||||
|
|
||||||
|
### `adapters`
|
||||||
|
|
||||||
|
`[]Adapter`
|
||||||
|
|
||||||
|
Adapters are LoRA adapters served on this model's replicas, each asked
|
||||||
|
for by its own name as though it were a model. Every replica loads
|
||||||
|
every adapter, applying one only to a request that names it.
|
||||||
|
|
||||||
### `fallbacks`
|
### `fallbacks`
|
||||||
|
|
||||||
`[]string`
|
`[]string`
|
||||||
@@ -938,6 +1069,20 @@ this. A node that does not report its memory is not excluded either —
|
|||||||
an older agent must not have its nodes quietly emptied by an upgrade of
|
an older agent must not have its nodes quietly emptied by an upgrade of
|
||||||
the control plane.
|
the control plane.
|
||||||
|
|
||||||
|
### `cpu_cores`
|
||||||
|
|
||||||
|
`int`
|
||||||
|
|
||||||
|
CPUCores is the cores a replica of a profile with no GPU is given:
|
||||||
|
reserved out of its node's CPUs by placement, as memory is, and what
|
||||||
|
its engine computes on (llama.cpp's --threads).
|
||||||
|
|
||||||
|
Reserved because an engine takes every core it is told about, and two
|
||||||
|
on one node that each think they have all of them spin against each
|
||||||
|
other: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second
|
||||||
|
to under one each. Zero reserves nothing and leaves the thread count to
|
||||||
|
the node, which is every profile from before this.
|
||||||
|
|
||||||
### `min_disk_gb`
|
### `min_disk_gb`
|
||||||
|
|
||||||
`int`
|
`int`
|
||||||
@@ -1236,6 +1381,16 @@ Monitoring is where the dashboard reads its history from.
|
|||||||
|
|
||||||
Takes a [Monitoring](#monitoring) block.
|
Takes a [Monitoring](#monitoring) block.
|
||||||
|
|
||||||
|
### `tracing`
|
||||||
|
|
||||||
|
`Tracing`
|
||||||
|
|
||||||
|
Tracing sends a trace of each inference request to an OpenTelemetry
|
||||||
|
collector. Off unless an endpoint is set here or in
|
||||||
|
OTEL_EXPORTER_OTLP_ENDPOINT.
|
||||||
|
|
||||||
|
Takes a [Tracing](#tracing) block.
|
||||||
|
|
||||||
### `mirror`
|
### `mirror`
|
||||||
|
|
||||||
`Mirror`
|
`Mirror`
|
||||||
@@ -1257,6 +1412,22 @@ never trusted to already have what somebody once put there.
|
|||||||
Unset keeps DefaultEngines. A version replaces the default; an empty
|
Unset keeps DefaultEngines. A version replaces the default; an empty
|
||||||
one leaves that engine to be installed by hand.
|
one leaves that engine to be installed by hand.
|
||||||
|
|
||||||
|
### `backup_dir`
|
||||||
|
|
||||||
|
`string`
|
||||||
|
|
||||||
|
BackupDir is where `llmgw server backup` writes, and where the newest
|
||||||
|
archive's age is read for llmgw_backup_last_success_timestamp_seconds.
|
||||||
|
Empty means /var/backups/llmgw-server.
|
||||||
|
|
||||||
|
### `releases`
|
||||||
|
|
||||||
|
`string`
|
||||||
|
|
||||||
|
Releases is where `llmgw ctl upgrade` fetches release binaries from,
|
||||||
|
for this machine and — through their agents — every node. Empty means
|
||||||
|
the project's own releases.
|
||||||
|
|
||||||
### `router`
|
### `router`
|
||||||
|
|
||||||
`Router`
|
`Router`
|
||||||
@@ -1340,6 +1511,22 @@ about a hundred tokens on a slow model.
|
|||||||
|
|
||||||
`time.Duration`
|
`time.Duration`
|
||||||
|
|
||||||
|
## Tracing
|
||||||
|
|
||||||
|
### `endpoint`
|
||||||
|
|
||||||
|
`string`
|
||||||
|
|
||||||
|
Endpoint is the collector, for example http://otel.internal:4318.
|
||||||
|
Empty falls back to OTEL_EXPORTER_OTLP_ENDPOINT, and to no tracing.
|
||||||
|
|
||||||
|
### `sample_ratio`
|
||||||
|
|
||||||
|
`float64`
|
||||||
|
|
||||||
|
SampleRatio is the share of requests traced, 0 to 1, when the caller
|
||||||
|
did not decide it with a sampled traceparent of its own. Zero takes 1.
|
||||||
|
|
||||||
## VLLMConfig
|
## VLLMConfig
|
||||||
|
|
||||||
### `bin`
|
### `bin`
|
||||||
|
|||||||
@@ -19,6 +19,10 @@ require (
|
|||||||
github.com/prometheus/common v0.71.0
|
github.com/prometheus/common v0.71.0
|
||||||
github.com/spf13/cobra v1.10.2
|
github.com/spf13/cobra v1.10.2
|
||||||
github.com/yuin/goldmark v1.8.6
|
github.com/yuin/goldmark v1.8.6
|
||||||
|
go.opentelemetry.io/otel v1.46.0
|
||||||
|
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0
|
||||||
|
go.opentelemetry.io/otel/sdk v1.46.0
|
||||||
|
go.opentelemetry.io/otel/trace v1.46.0
|
||||||
golang.org/x/crypto v0.57.0
|
golang.org/x/crypto v0.57.0
|
||||||
golang.org/x/net v0.59.0
|
golang.org/x/net v0.59.0
|
||||||
golang.org/x/sync v0.23.0
|
golang.org/x/sync v0.23.0
|
||||||
@@ -29,6 +33,7 @@ require (
|
|||||||
require (
|
require (
|
||||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||||
github.com/beorn7/perks v1.0.1 // indirect
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
|
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||||
github.com/charmbracelet/colorprofile v0.4.1 // indirect
|
github.com/charmbracelet/colorprofile v0.4.1 // indirect
|
||||||
github.com/charmbracelet/x/ansi v0.11.6 // indirect
|
github.com/charmbracelet/x/ansi v0.11.6 // indirect
|
||||||
github.com/charmbracelet/x/cellbuf v0.0.15 // indirect
|
github.com/charmbracelet/x/cellbuf v0.0.15 // indirect
|
||||||
@@ -39,8 +44,11 @@ require (
|
|||||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
github.com/go-logr/logr v1.4.4 // indirect
|
||||||
|
github.com/go-logr/stdr v1.2.2 // indirect
|
||||||
|
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||||
github.com/google/uuid v1.6.0 // indirect
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
|
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||||
@@ -61,9 +69,16 @@ require (
|
|||||||
github.com/rivo/uniseg v0.4.7 // indirect
|
github.com/rivo/uniseg v0.4.7 // indirect
|
||||||
github.com/spf13/pflag v1.0.9 // indirect
|
github.com/spf13/pflag v1.0.9 // indirect
|
||||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||||
|
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||||
|
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect
|
||||||
|
go.opentelemetry.io/otel/metric v1.46.0 // indirect
|
||||||
|
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||||
golang.org/x/sys v0.48.0 // indirect
|
golang.org/x/sys v0.48.0 // indirect
|
||||||
golang.org/x/text v0.42.0 // indirect
|
golang.org/x/text v0.42.0 // indirect
|
||||||
|
google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect
|
||||||
|
google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect
|
||||||
|
google.golang.org/grpc v1.83.1 // indirect
|
||||||
modernc.org/libc v1.75.7 // indirect
|
modernc.org/libc v1.75.7 // indirect
|
||||||
modernc.org/mathutil v1.7.1 // indirect
|
modernc.org/mathutil v1.7.1 // indirect
|
||||||
modernc.org/memory v1.12.1 // indirect
|
modernc.org/memory v1.12.1 // indirect
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE
|
|||||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||||
|
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
|
||||||
|
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
|
||||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
|
github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
|
||||||
@@ -34,14 +36,23 @@ github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6
|
|||||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
|
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
|
||||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
|
||||||
|
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||||
|
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||||
|
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||||
|
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
|
||||||
|
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||||
|
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||||
|
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
|
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
|
||||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
|
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
|
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
|
||||||
|
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
|
||||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||||
@@ -117,6 +128,24 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM
|
|||||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||||
github.com/yuin/goldmark v1.8.6 h1:d0VcaP1sx9GkFVkoW+KtggpGi2KZ965i14b0+bDQST4=
|
github.com/yuin/goldmark v1.8.6 h1:d0VcaP1sx9GkFVkoW+KtggpGi2KZ965i14b0+bDQST4=
|
||||||
github.com/yuin/goldmark v1.8.6/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
github.com/yuin/goldmark v1.8.6/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
||||||
|
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||||
|
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||||
|
go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
|
||||||
|
go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE=
|
||||||
|
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw=
|
||||||
|
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g=
|
||||||
|
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 h1:KrC1YrQeSt46ITMWAbgQx1M1eV1/1TKzttrBzymPmss=
|
||||||
|
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0/go.mod h1:zDSEzoEqsOrgBeGvH66KRgxh90VonFyJqBHA0Pk3+rM=
|
||||||
|
go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8=
|
||||||
|
go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o=
|
||||||
|
go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI=
|
||||||
|
go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM=
|
||||||
|
go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE=
|
||||||
|
go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4=
|
||||||
|
go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c=
|
||||||
|
go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI=
|
||||||
|
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
|
||||||
|
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
|
||||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||||
@@ -141,6 +170,14 @@ golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
|||||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||||
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
||||||
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
||||||
|
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||||
|
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||||
|
google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI=
|
||||||
|
google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI=
|
||||||
|
google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 h1:cYNAzI2sUwhmCcoj9TxvihSrqsxt6uIkj3rDRhSDmW4=
|
||||||
|
google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA=
|
||||||
|
google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y=
|
||||||
|
google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
|
||||||
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||||
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"golang.org/x/crypto/argon2"
|
"golang.org/x/crypto/argon2"
|
||||||
@@ -56,7 +57,7 @@ func HashPassword(password string) (string, error) {
|
|||||||
if _, err := rand.Read(salt); err != nil {
|
if _, err := rand.Read(salt); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
sum := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen)
|
sum := idKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen)
|
||||||
|
|
||||||
b64 := base64.RawStdEncoding.EncodeToString
|
b64 := base64.RawStdEncoding.EncodeToString
|
||||||
return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s",
|
return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s",
|
||||||
@@ -93,7 +94,7 @@ func VerifyPassword(password, encoded string) (bool, error) {
|
|||||||
return false, ErrBadHash
|
return false, ErrBadHash
|
||||||
}
|
}
|
||||||
|
|
||||||
got := argon2.IDKey([]byte(password), salt, time, memory, threads, uint32(len(want)))
|
got := idKey([]byte(password), salt, time, memory, threads, uint32(len(want)))
|
||||||
return subtle.ConstantTimeCompare(got, want) == 1, nil
|
return subtle.ConstantTimeCompare(got, want) == 1, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -110,3 +111,16 @@ func init() {
|
|||||||
}
|
}
|
||||||
dummyHash = h
|
dummyHash = h
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// argonSlots bounds how many hashes run at once. Each takes 64 MiB, and the
|
||||||
|
// login and signup forms are on the client-facing port: without a bound, a
|
||||||
|
// few hundred parallel attempts at random addresses asked for gigabytes at
|
||||||
|
// once, and the per-address throttle does not see them — each address is new.
|
||||||
|
// Beyond the bound, attempts wait their turn instead.
|
||||||
|
var argonSlots = make(chan struct{}, max(2, runtime.NumCPU()/2))
|
||||||
|
|
||||||
|
func idKey(password, salt []byte, time, memory uint32, threads uint8, keyLen uint32) []byte {
|
||||||
|
argonSlots <- struct{}{}
|
||||||
|
defer func() { <-argonSlots }()
|
||||||
|
return argon2.IDKey(password, salt, time, memory, threads, keyLen)
|
||||||
|
}
|
||||||
@@ -230,6 +230,22 @@ func (s *Service) allowAttempt(email string) bool {
|
|||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
w, ok := s.attempts[email]
|
w, ok := s.attempts[email]
|
||||||
|
if !ok && len(s.attempts) >= maxTrackedAddresses {
|
||||||
|
// Full of addresses nobody has signed in as for a while: forget the
|
||||||
|
// ones whose window has passed. Only expiring, never evicting early —
|
||||||
|
// dropping a live window is how an attacker spraying random addresses
|
||||||
|
// would buy unlimited guesses at the one they care about.
|
||||||
|
for k, v := range s.attempts {
|
||||||
|
if now.Sub(v.since) > s.cfg.AttemptWindow {
|
||||||
|
delete(s.attempts, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(s.attempts) >= maxTrackedAddresses {
|
||||||
|
// Still full, so somebody is spraying addresses faster than they
|
||||||
|
// expire. New ones wait; those already tracked are unaffected.
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
if !ok || now.Sub(w.since) > s.cfg.AttemptWindow {
|
if !ok || now.Sub(w.since) > s.cfg.AttemptWindow {
|
||||||
s.attempts[email] = &attemptWindow{count: 1, since: now}
|
s.attempts[email] = &attemptWindow{count: 1, since: now}
|
||||||
return true
|
return true
|
||||||
@@ -238,6 +254,10 @@ func (s *Service) allowAttempt(email string) bool {
|
|||||||
return w.count <= s.cfg.MaxAttempts
|
return w.count <= s.cfg.MaxAttempts
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// maxTrackedAddresses bounds the throttle's memory. Unbounded, it grew by one
|
||||||
|
// entry per address tried and shrank only on a successful sign-in.
|
||||||
|
const maxTrackedAddresses = 50000
|
||||||
|
|
||||||
func (s *Service) clearAttempts(email string) {
|
func (s *Service) clearAttempts(email string) {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
delete(s.attempts, email)
|
delete(s.attempts, email)
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package account
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSprayingAddressesCannotGrowTheThrottleOrEscapeIt(t *testing.T) {
|
||||||
|
now := time.Unix(0, 0)
|
||||||
|
s := &Service{cfg: DefaultConfig(), attempts: map[string]*attemptWindow{}, now: func() time.Time { return now }}
|
||||||
|
|
||||||
|
// The target is being guessed at.
|
||||||
|
for i := 0; i < s.cfg.MaxAttempts; i++ {
|
||||||
|
s.allowAttempt("victim@example.com")
|
||||||
|
}
|
||||||
|
// Then the table is flooded with made-up addresses.
|
||||||
|
for i := 0; i < maxTrackedAddresses+100; i++ {
|
||||||
|
s.allowAttempt(fmt.Sprintf("x%d@example.com", i))
|
||||||
|
}
|
||||||
|
if n := len(s.attempts); n > maxTrackedAddresses {
|
||||||
|
t.Errorf("the table grew to %d", n)
|
||||||
|
}
|
||||||
|
if s.allowAttempt("victim@example.com") {
|
||||||
|
t.Error("flooding the table reset the target's throttle")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Once the window passes, everything expires and sign-ins work again.
|
||||||
|
now = now.Add(s.cfg.AttemptWindow + time.Second)
|
||||||
|
if !s.allowAttempt("new@example.com") {
|
||||||
|
t.Error("a new address was refused after the flood expired")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -383,3 +383,39 @@ func keyCounter(keyID string) string {
|
|||||||
// they are asking is whether the budgets they are about to display mean
|
// they are asking is whether the budgets they are about to display mean
|
||||||
// anything, and the accountant is what would enforce them.
|
// anything, and the accountant is what would enforce them.
|
||||||
func (a *Accountant) AnythingPriced() bool { return a.pricing.AnythingPriced() }
|
func (a *Accountant) AnythingPriced() bool { return a.pricing.AnythingPriced() }
|
||||||
|
|
||||||
|
// Headroom is the tightest of a principal's request and token limits: how
|
||||||
|
// much is left of each, and when the minute it is counted over rolls on.
|
||||||
|
type Headroom struct {
|
||||||
|
RequestLimit, RequestsLeft int
|
||||||
|
TokenLimit, TokensLeft int
|
||||||
|
Reset time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// Headroom reports what a principal has left, after whatever Check just
|
||||||
|
// charged. Of several levels the one with least left is the one that will
|
||||||
|
// refuse first, so that is the one reported. A limit of 0 means none is set.
|
||||||
|
func (a *Accountant) Headroom(p auth.Principal) Headroom {
|
||||||
|
var h Headroom
|
||||||
|
firstReq, firstTok := true, true
|
||||||
|
for _, l := range levels(p.Limits, p.KeyID) {
|
||||||
|
if l.RPM != nil && *l.RPM > 0 {
|
||||||
|
left := max(0, *l.RPM-int(a.requests.Rate(l.Counter)))
|
||||||
|
if firstReq || left < h.RequestsLeft {
|
||||||
|
h.RequestLimit, h.RequestsLeft, firstReq = *l.RPM, left, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if l.TPM != nil && *l.TPM > 0 {
|
||||||
|
left := max(0, *l.TPM-int(a.tokens.Rate(l.Counter)))
|
||||||
|
if firstTok || left < h.TokensLeft {
|
||||||
|
h.TokenLimit, h.TokensLeft, firstTok = *l.TPM, left, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The windows slide, so room comes back continuously; the next minute
|
||||||
|
// boundary is when a whole window's worth has moved on, and is the figure
|
||||||
|
// a client pacing itself can act on.
|
||||||
|
now := time.Now()
|
||||||
|
h.Reset = now.Truncate(windowLen).Add(windowLen).Sub(now)
|
||||||
|
return h
|
||||||
|
}
|
||||||
@@ -630,3 +630,31 @@ func TestAnEmptyPriceTableMeansNoBudgetCanEverBind(t *testing.T) {
|
|||||||
t.Error("no price table at all is not priced")
|
t.Error("no price table at all is not priced")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHeadroomIsTheTightestLimitAfterThisRequest(t *testing.T) {
|
||||||
|
// A project allowing 100 a minute under a team allowing 3: the team is
|
||||||
|
// what will refuse, so the team is what a client is told about.
|
||||||
|
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||||
|
p := principal("k1", func(e *store.Effective) {
|
||||||
|
e.Chain = []store.Limits{
|
||||||
|
{Scope: store.ScopeProject, ScopeID: "proj", RPM: intp(100), TPM: intp(50000)},
|
||||||
|
{Scope: store.ScopeTeam, ScopeID: "team", RPM: intp(3)},
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if _, err := a.Check(p, "m", 10); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
h := a.Headroom(p)
|
||||||
|
if h.RequestLimit != 3 || h.RequestsLeft != 2 {
|
||||||
|
t.Errorf("requests %d left of %d; expected 2 of the team's 3", h.RequestsLeft, h.RequestLimit)
|
||||||
|
}
|
||||||
|
if h.TokenLimit != 50000 || h.TokensLeft != 50000-10 {
|
||||||
|
t.Errorf("tokens %d left of %d", h.TokensLeft, h.TokenLimit)
|
||||||
|
}
|
||||||
|
if h.Reset <= 0 || h.Reset > time.Minute {
|
||||||
|
t.Errorf("reset %v", h.Reset)
|
||||||
|
}
|
||||||
|
if none := a.Headroom(principal("k2", nil)); none.RequestLimit != 0 || none.TokenLimit != 0 {
|
||||||
|
t.Errorf("a key with no limits advertised some: %+v", none)
|
||||||
|
}
|
||||||
|
}
|
||||||
+29
-812
@@ -14,13 +14,10 @@ import (
|
|||||||
"math/rand/v2"
|
"math/rand/v2"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"slices"
|
"slices"
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -34,278 +31,9 @@ import (
|
|||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config configures the agent.
|
|
||||||
type Config struct {
|
|
||||||
// NodeID defaults to the hostname. It must equal the common name of the
|
|
||||||
// agent's certificate, so set it only where the two differ.
|
|
||||||
NodeID string `koanf:"node_id"`
|
|
||||||
// Server is the control plane, and Servers is the same thing when there is
|
|
||||||
// more than one. Both are accepted: every configuration written before
|
|
||||||
// there could be a second control plane names one address, and none of
|
|
||||||
// them should have to change.
|
|
||||||
//
|
|
||||||
// The order is tried in order, and a refusal moves to the next. Only the
|
|
||||||
// leader accepts an agent, so a standby is skipped in the time it takes to
|
|
||||||
// be told so — which is what makes failover a reconnect rather than an
|
|
||||||
// edit to every node's configuration.
|
|
||||||
Server string `koanf:"server"`
|
|
||||||
Servers []string `koanf:"servers"`
|
|
||||||
// Hostname defaults to the machine's.
|
|
||||||
Hostname string `koanf:"hostname"`
|
|
||||||
// MgmtAddr is the address engines are reached on. It defaults to this
|
|
||||||
// machine's address on the route to the control plane, which is the one
|
|
||||||
// the control plane and the router can reach.
|
|
||||||
MgmtAddr string `koanf:"mgmt_addr"`
|
|
||||||
// Labels are free-form facts about a node for a profile's node_selector.
|
|
||||||
// Nothing needs them: memory, GPUs, engines and disk are measured, and a
|
|
||||||
// label restating one of those goes stale the day the machine changes.
|
|
||||||
Labels map[string]string `koanf:"labels"`
|
|
||||||
|
|
||||||
// MgmtInterface is the ethernet interface NCCL bootstraps over. It
|
|
||||||
// defaults to the interface holding MgmtAddr, then eth0.
|
|
||||||
MgmtInterface string `koanf:"mgmt_interface"`
|
|
||||||
// Fabrics labels this node's HCAs. It is explicit on purpose: OpenSM's
|
|
||||||
// default subnet prefix is the same on every fabric, so nothing can be
|
|
||||||
// inferred from the wire (plan §5.2).
|
|
||||||
Fabrics []inventory.FabricAttachment `koanf:"fabrics"`
|
|
||||||
|
|
||||||
HeartbeatInterval time.Duration `koanf:"heartbeat_interval"`
|
|
||||||
// ReconnectBase is the first reconnect delay; it backs off from there.
|
|
||||||
ReconnectBase time.Duration `koanf:"reconnect_base"`
|
|
||||||
ReconnectMax time.Duration `koanf:"reconnect_max"`
|
|
||||||
|
|
||||||
// AdoptReadyTimeout bounds how long an adopted engine is given to answer a
|
|
||||||
// health check before the agent stops waiting on it.
|
|
||||||
//
|
|
||||||
// Shorter than a startup timeout on purpose: this engine is already
|
|
||||||
// running, so it is not loading a model, it is either serving or it is not.
|
|
||||||
AdoptReadyTimeout time.Duration `koanf:"adopt_ready_timeout"`
|
|
||||||
|
|
||||||
// TLS authenticates both ends of the control-plane stream. Leaving it unset
|
|
||||||
// keeps the plaintext behaviour, which only belongs on a trusted network.
|
|
||||||
TLS tlsconf.Config `koanf:"tls"`
|
|
||||||
|
|
||||||
// ModelDirs are the directories this node keeps model weights in, beyond
|
|
||||||
// WeightsCache, which is always looked in. Storage differs at every installation,
|
|
||||||
// so llmgw observes it rather than owning it: a Lustre mount, an image
|
|
||||||
// exported over the fabric, local NVMe and a copy made by hand all look
|
|
||||||
// identical from here, and placement only needs to know whether the file
|
|
||||||
// is present.
|
|
||||||
ModelDirs []string `koanf:"model_dirs"`
|
|
||||||
|
|
||||||
// EnginesDir is where the agent installs engines, one directory per
|
|
||||||
// build. Defaults to /opt/llmgw/engines.
|
|
||||||
EnginesDir string `koanf:"engines_dir"`
|
|
||||||
|
|
||||||
// WeightsCache is where a remote source is downloaded to. Defaults to the
|
|
||||||
// first entry of ModelDirs, because a node that says where its models live
|
|
||||||
// has already answered this question, and otherwise to
|
|
||||||
// /var/lib/llmgw-weights.
|
|
||||||
//
|
|
||||||
// Fetching is the simple arrangement: every node keeps its own copy. The
|
|
||||||
// exotic ones need no code — weights on a shared filer or an image exported
|
|
||||||
// over the fabric are simply present, and present means nothing is fetched.
|
|
||||||
WeightsCache string `koanf:"weights_cache"`
|
|
||||||
|
|
||||||
Discovery discovery.Options `koanf:"discovery"`
|
|
||||||
Engines engines.Config `koanf:"engines"`
|
|
||||||
Superv supervisor.Config `koanf:"supervisor"`
|
|
||||||
|
|
||||||
Version string
|
|
||||||
}
|
|
||||||
|
|
||||||
// DefaultConfig returns sensible defaults.
|
|
||||||
func DefaultConfig() Config {
|
|
||||||
// No MgmtInterface: normalize finds the one holding the management
|
|
||||||
// address, and falls back to eth0 only when it cannot. Defaulted here, it
|
|
||||||
// was never found — every node said eth0 unless told otherwise, and a node
|
|
||||||
// on ens3 or bond0 had NCCL bootstrap over an interface it does not have.
|
|
||||||
return Config{
|
|
||||||
HeartbeatInterval: 5 * time.Second,
|
|
||||||
ReconnectBase: time.Second,
|
|
||||||
ReconnectMax: 30 * time.Second,
|
|
||||||
AdoptReadyTimeout: 60 * time.Second,
|
|
||||||
Superv: supervisor.DefaultConfig(),
|
|
||||||
// NVML gives NVLink topology, so prefer it when this build has it.
|
|
||||||
Discovery: discovery.Options{PreferNVML: true},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Config) normalize() error {
|
|
||||||
d := DefaultConfig()
|
|
||||||
if err := c.TLS.Validate(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := c.normalizeServerURL(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// Everything below is found rather than written. A file that has to say
|
|
||||||
// which machine it is on, and at what address, is a file that differs on
|
|
||||||
// every node — and a copy taken from one node to the next registers the
|
|
||||||
// second as the first.
|
|
||||||
host, _ := os.Hostname()
|
|
||||||
if c.NodeID == "" {
|
|
||||||
c.NodeID = host
|
|
||||||
}
|
|
||||||
if c.NodeID == "" {
|
|
||||||
return errors.New("node_id is not set and the hostname could not be read")
|
|
||||||
}
|
|
||||||
if c.Hostname == "" {
|
|
||||||
c.Hostname = host
|
|
||||||
}
|
|
||||||
if c.MgmtAddr == "" {
|
|
||||||
c.MgmtAddr = addrToward(c.Server)
|
|
||||||
}
|
|
||||||
if c.MgmtInterface == "" {
|
|
||||||
c.MgmtInterface = interfaceOf(c.MgmtAddr)
|
|
||||||
}
|
|
||||||
if c.MgmtInterface == "" {
|
|
||||||
c.MgmtInterface = "eth0"
|
|
||||||
}
|
|
||||||
// The directory downloads land in is always one that is looked in. When
|
|
||||||
// they were two settings, a node could fill its disk with fetched weights
|
|
||||||
// that placement never saw — which is what every node here was doing.
|
|
||||||
if c.WeightsCache == "" && len(c.ModelDirs) > 0 {
|
|
||||||
c.WeightsCache = c.ModelDirs[0]
|
|
||||||
}
|
|
||||||
if c.WeightsCache == "" {
|
|
||||||
c.WeightsCache = DefaultWeightsCache
|
|
||||||
}
|
|
||||||
if !slices.Contains(c.ModelDirs, c.WeightsCache) {
|
|
||||||
c.ModelDirs = append(c.ModelDirs, c.WeightsCache)
|
|
||||||
}
|
|
||||||
if c.HeartbeatInterval <= 0 {
|
|
||||||
c.HeartbeatInterval = d.HeartbeatInterval
|
|
||||||
}
|
|
||||||
if c.ReconnectBase <= 0 {
|
|
||||||
c.ReconnectBase = d.ReconnectBase
|
|
||||||
}
|
|
||||||
if c.AdoptReadyTimeout <= 0 {
|
|
||||||
c.AdoptReadyTimeout = d.AdoptReadyTimeout
|
|
||||||
}
|
|
||||||
if c.ReconnectMax < c.ReconnectBase {
|
|
||||||
c.ReconnectMax = max(c.ReconnectBase, d.ReconnectMax)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// normalizeServerURL makes the configured address a URL connect can use, and
|
|
||||||
// refuses the combination that would quietly undo the certificates: a scheme
|
|
||||||
// that says plaintext while TLS is configured. Silently connecting anyway would
|
|
||||||
// mean a cluster believing it has mutual authentication when it has none.
|
|
||||||
func (c *Config) normalizeServerURL() error {
|
|
||||||
// One list, however it was written. Server first when both are given,
|
|
||||||
// because an operator adding a standby to an existing file expects the
|
|
||||||
// one already there to stay preferred.
|
|
||||||
all := make([]string, 0, 1+len(c.Servers))
|
|
||||||
if c.Server != "" {
|
|
||||||
all = append(all, c.Server)
|
|
||||||
}
|
|
||||||
all = append(all, c.Servers...)
|
|
||||||
|
|
||||||
seen := map[string]struct{}{}
|
|
||||||
out := make([]string, 0, len(all))
|
|
||||||
for _, s := range all {
|
|
||||||
fixed, err := normalizeOne(s, c.TLS.Enabled())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// A duplicate would be tried twice in a row on failover, which reads
|
|
||||||
// as the rotation not working.
|
|
||||||
if _, dup := seen[fixed]; dup {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
seen[fixed] = struct{}{}
|
|
||||||
out = append(out, fixed)
|
|
||||||
}
|
|
||||||
if len(out) == 0 {
|
|
||||||
return errors.New("server is required")
|
|
||||||
}
|
|
||||||
c.Servers = out
|
|
||||||
c.Server = out[0]
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// DefaultWeightsCache is where fetched weights go when nothing says otherwise.
|
|
||||||
const DefaultWeightsCache = "/var/lib/llmgw-weights"
|
|
||||||
|
|
||||||
// addrToward is this machine's address on the route to the control plane.
|
|
||||||
//
|
|
||||||
// The address engines are reached on is whichever one the control plane can
|
|
||||||
// reach, and the kernel already knows which that is: a UDP "connection" picks
|
|
||||||
// the source address for the route without sending a packet. Asking the
|
|
||||||
// interfaces instead would have to guess among docker bridges, VPNs and
|
|
||||||
// link-local addresses.
|
|
||||||
func addrToward(server string) string {
|
|
||||||
u, err := url.Parse(server)
|
|
||||||
if err != nil || u.Host == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
host := u.Hostname()
|
|
||||||
port := u.Port()
|
|
||||||
if port == "" {
|
|
||||||
port = "443"
|
|
||||||
}
|
|
||||||
conn, err := net.Dial("udp", net.JoinHostPort(host, port))
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
defer conn.Close()
|
|
||||||
if a, ok := conn.LocalAddr().(*net.UDPAddr); ok {
|
|
||||||
return a.IP.String()
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// interfaceOf names the interface that holds an address, or "" when none does.
|
|
||||||
func interfaceOf(addr string) string {
|
|
||||||
ip := net.ParseIP(addr)
|
|
||||||
if ip == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
ifaces, err := net.Interfaces()
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
for _, ifc := range ifaces {
|
|
||||||
addrs, err := ifc.Addrs()
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, a := range addrs {
|
|
||||||
if n, ok := a.(*net.IPNet); ok && n.IP.Equal(ip) {
|
|
||||||
return ifc.Name
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// normalizeOne applies the scheme rules to a single address.
|
|
||||||
func normalizeOne(addr string, secure bool) (string, error) {
|
|
||||||
switch {
|
|
||||||
case strings.HasPrefix(addr, "https://"):
|
|
||||||
if !secure {
|
|
||||||
return "", fmt.Errorf("%s is an https:// address but no tls certificates are configured", addr)
|
|
||||||
}
|
|
||||||
case strings.HasPrefix(addr, "http://"):
|
|
||||||
if secure {
|
|
||||||
return "", fmt.Errorf("tls is configured but %s is an http:// address; use https://", addr)
|
|
||||||
}
|
|
||||||
case strings.Contains(addr, "://"):
|
|
||||||
return "", fmt.Errorf("server %q must be an http:// or https:// address", addr)
|
|
||||||
case secure:
|
|
||||||
return "https://" + addr, nil
|
|
||||||
default:
|
|
||||||
return "http://" + addr, nil
|
|
||||||
}
|
|
||||||
return addr, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Agent runs on a GPU node.
|
// Agent runs on a GPU node.
|
||||||
type Agent struct {
|
type Agent struct {
|
||||||
cfg Config
|
cfg Config
|
||||||
@@ -373,6 +101,9 @@ type Agent struct {
|
|||||||
smsErr map[string]string
|
smsErr map[string]string
|
||||||
// load measures the machine between heartbeats.
|
// load measures the machine between heartbeats.
|
||||||
load *discovery.LoadSampler
|
load *discovery.LoadSampler
|
||||||
|
// sensors reads heat and power: CPU and board from hwmon and RAPL, GPUs
|
||||||
|
// from nvidia-smi.
|
||||||
|
sensors *discovery.SensorSampler
|
||||||
|
|
||||||
// fetchWatchers are the shards waiting on each source, so progress on one
|
// fetchWatchers are the shards waiting on each source, so progress on one
|
||||||
// download reaches every shard that asked for it. A fetch is single-flight
|
// download reaches every shard that asked for it. A fetch is single-flight
|
||||||
@@ -418,6 +149,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
|
|||||||
logRing: ring,
|
logRing: ring,
|
||||||
provKick: make(chan struct{}, 1),
|
provKick: make(chan struct{}, 1),
|
||||||
load: discovery.NewLoadSampler(),
|
load: discovery.NewLoadSampler(),
|
||||||
|
sensors: discovery.NewSensorSampler(),
|
||||||
}
|
}
|
||||||
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
|
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
|
||||||
return a, nil
|
return a, nil
|
||||||
@@ -586,131 +318,6 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) {
|
|||||||
"shard", o.ShardID, "replica", o.ReplicaID)
|
"shard", o.ShardID, "replica", o.ReplicaID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// usableFabrics reports the fabrics this node can actually carry traffic on.
|
|
||||||
//
|
|
||||||
// Loud about what it drops. A fabric silently missing from a registration looks
|
|
||||||
// identical to one never configured, and the operator who typed the adapter
|
|
||||||
// name is the only person who can fix it — so the reason is logged here, on the
|
|
||||||
// machine where both halves are visible, rather than inferred later from a
|
|
||||||
// replica that will not converge.
|
|
||||||
func (a *Agent) usableFabrics(inv discovery.Inventory) []inventory.FabricAttachment {
|
|
||||||
out := a.checkFabrics(inv.HCAs, true)
|
|
||||||
a.mu.Lock()
|
|
||||||
a.fabricsSent = fabricsFingerprint(out)
|
|
||||||
a.mu.Unlock()
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkFabrics is the declared fabrics this node can carry traffic on, from
|
|
||||||
// the adapters as they are now. Loud logs every problem; the heartbeat checks
|
|
||||||
// quietly and speaks only when the answer changes.
|
|
||||||
func (a *Agent) checkFabrics(hcas []discovery.HCA, loud bool) []inventory.FabricAttachment {
|
|
||||||
declared := map[string][]string{}
|
|
||||||
for _, f := range a.cfg.Fabrics {
|
|
||||||
declared[f.Fabric] = f.HCAs
|
|
||||||
}
|
|
||||||
|
|
||||||
var out []inventory.FabricAttachment
|
|
||||||
for _, c := range discovery.CheckFabrics(declared, hcas) {
|
|
||||||
if loud {
|
|
||||||
for _, p := range c.Problems {
|
|
||||||
a.log.Error("a declared fabric adapter cannot carry traffic",
|
|
||||||
"fabric", c.Fabric, "problem", p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !c.OK() {
|
|
||||||
if loud {
|
|
||||||
a.log.Error("fabric not advertised; no declared adapter on it can carry traffic",
|
|
||||||
"fabric", c.Fabric,
|
|
||||||
"consequence", "no multi-node replica will be placed here")
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, inventory.FabricAttachment{
|
|
||||||
Fabric: c.Fabric, HCAs: c.Usable, RateGbps: c.RateGbps,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// fabricsFingerprint is enough to notice a fabric, an adapter or a rate
|
|
||||||
// changing.
|
|
||||||
func fabricsFingerprint(fs []inventory.FabricAttachment) string {
|
|
||||||
var b strings.Builder
|
|
||||||
for _, f := range fs {
|
|
||||||
fmt.Fprintf(&b, "%s:%s:%d;", f.Fabric, strings.Join(f.HCAs, ","), f.RateGbps)
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// sampleIB re-reads the adapters for the heartbeat: every port's state and
|
|
||||||
// counters, and the fabrics again when they no longer match what the control
|
|
||||||
// plane was told.
|
|
||||||
//
|
|
||||||
// Every heartbeat, because it is a few small files per port, and because a
|
|
||||||
// port that went down used to be invisible until the agent restarted — the
|
|
||||||
// node went on being offered multi-node work that would hang on its first
|
|
||||||
// collective.
|
|
||||||
func (a *Agent) sampleIB(hb *llmgwv1.Heartbeat) {
|
|
||||||
a.mu.Lock()
|
|
||||||
had := len(a.inv.HCAs) > 0
|
|
||||||
a.mu.Unlock()
|
|
||||||
if !had && len(a.cfg.Fabrics) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
hcas, err := discovery.DiscoverIB(a.cfg.Discovery.IBRoot)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.attachSM(hcas)
|
|
||||||
for _, h := range hcas {
|
|
||||||
for _, p := range h.Ports {
|
|
||||||
ip := &llmgwv1.IBPort{
|
|
||||||
Hca: h.Name, Port: int32(p.Num), State: p.State, PhysState: p.PhysState,
|
|
||||||
RateGbps: int32(p.RateGbps), LinkLayer: string(p.LinkLayer), Counters: p.Counters,
|
|
||||||
NodeGuid: h.NodeGUID, PortGuid: p.PortGUID, Network: p.Network, GidPrefix: p.GIDPrefix,
|
|
||||||
}
|
|
||||||
a.mu.Lock()
|
|
||||||
if p.SM != nil {
|
|
||||||
ip.SmGuid, ip.SmLid, ip.SmPriority, ip.SmState = p.SM.GUID, int32(p.SM.LID), int32(p.SM.Priority), p.SM.State
|
|
||||||
} else {
|
|
||||||
ip.SmError = a.smErr[smKey(h.Name, p.Num)]
|
|
||||||
}
|
|
||||||
for _, r := range a.sms[smKey(h.Name, p.Num)] {
|
|
||||||
ip.SubnetManagers = append(ip.SubnetManagers, &llmgwv1.SubnetManager{
|
|
||||||
Guid: r.GUID, Lid: int32(r.LID), Priority: int32(r.Priority), State: r.State})
|
|
||||||
}
|
|
||||||
ip.SubnetManagersError = a.smsErr[smKey(h.Name, p.Num)]
|
|
||||||
a.mu.Unlock()
|
|
||||||
hb.IbPorts = append(hb.IbPorts, ip)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
a.mu.Lock()
|
|
||||||
a.inv.HCAs = hcas // a reconnect registers what is true now
|
|
||||||
a.mu.Unlock()
|
|
||||||
if len(a.cfg.Fabrics) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
now := a.checkFabrics(hcas, false)
|
|
||||||
fp := fabricsFingerprint(now)
|
|
||||||
a.mu.Lock()
|
|
||||||
changed := fp != a.fabricsSent
|
|
||||||
a.fabricsSent = fp
|
|
||||||
a.mu.Unlock()
|
|
||||||
if !changed {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// Said once, with the reasons, the moment it happens.
|
|
||||||
a.checkFabrics(hcas, true)
|
|
||||||
a.log.Warn("the fabrics this node can carry traffic on changed", "now", fp)
|
|
||||||
hb.FabricsChanged = true
|
|
||||||
for _, f := range now {
|
|
||||||
hb.Fabrics = append(hb.Fabrics, &llmgwv1.FabricAttachment{
|
|
||||||
Fabric: f.Fabric, Hcas: f.HCAs, RateGbps: int32(f.RateGbps),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// weights builds the fetcher lazily, so an agent that never starts a shard
|
// weights builds the fetcher lazily, so an agent that never starts a shard
|
||||||
// never needs a cache directory.
|
// never needs a cache directory.
|
||||||
func (a *Agent) weights() *weights.Fetcher {
|
func (a *Agent) weights() *weights.Fetcher {
|
||||||
@@ -947,6 +554,7 @@ func (a *Agent) heartbeat() *llmgwv1.AgentMessage {
|
|||||||
// and a node that grew from 7 GB to 32 GB has to be seen to have grown
|
// and a node that grew from 7 GB to 32 GB has to be seen to have grown
|
||||||
// without anybody restarting anything.
|
// without anybody restarting anything.
|
||||||
hb.MemoryTotalMb = int64(discovery.MemoryTotalMB())
|
hb.MemoryTotalMb = int64(discovery.MemoryTotalMB())
|
||||||
|
hb.CpuCount = int32(runtime.NumCPU())
|
||||||
|
|
||||||
a.reprobeEngines()
|
a.reprobeEngines()
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
@@ -1034,6 +642,16 @@ func (a *Agent) sampleLoad() *llmgwv1.NodeLoad {
|
|||||||
ShardId: sh.ID, CpuCores: sh.CPUCores, MemoryRssBytes: sh.RSSBytes,
|
ShardId: sh.ID, CpuCores: sh.CPUCores, MemoryRssBytes: sh.RSSBytes,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
if a.sensors != nil {
|
||||||
|
se := a.sensors.Sample(context.Background())
|
||||||
|
out.CpuTempC, out.CpuPowerW, out.InletTempC = se.CPUTempC, se.CPUPowerW, se.InletTempC
|
||||||
|
for _, g := range se.GPUs {
|
||||||
|
out.Gpus = append(out.Gpus, &llmgwv1.GPUSensor{
|
||||||
|
Index: int32(g.Index), TempC: g.TempC, MemoryTempC: g.MemoryTempC,
|
||||||
|
PowerW: g.PowerW, PowerLimitW: g.PowerLimitW, Throttle: g.Throttle,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1119,6 +737,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
|
|||||||
a.removeWeights(m.RemoveWeights.GetPaths())
|
a.removeWeights(m.RemoveWeights.GetPaths())
|
||||||
case *llmgwv1.ControlMessage_Provision:
|
case *llmgwv1.ControlMessage_Provision:
|
||||||
a.setProvision(m.Provision)
|
a.setProvision(m.Provision)
|
||||||
|
case *llmgwv1.ControlMessage_Upgrade:
|
||||||
|
go a.upgrade(ctx, m.Upgrade, out)
|
||||||
case *llmgwv1.ControlMessage_Logs:
|
case *llmgwv1.ControlMessage_Logs:
|
||||||
reply := a.logsReply(m.Logs)
|
reply := a.logsReply(m.Logs)
|
||||||
// Off the receive loop: a full send queue must not stop commands being
|
// Off the receive loop: a full send queue must not stop commands being
|
||||||
@@ -1271,6 +891,17 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
|||||||
stop()
|
stop()
|
||||||
spec.Profile.ModelPath = path
|
spec.Profile.ModelPath = path
|
||||||
}
|
}
|
||||||
|
// Adapters are fetched like the model, and before the engine starts: it
|
||||||
|
// loads them at start, and one missing is a replica that answers for the
|
||||||
|
// base model under the adapter's name.
|
||||||
|
for i, ad := range spec.Profile.Adapters {
|
||||||
|
path, err := a.weights().Ensure(ctx, ad.Path)
|
||||||
|
if err != nil {
|
||||||
|
report(supervisor.Failed, fmt.Sprintf("adapter %s: %v", ad.Name, err), nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
spec.Profile.Adapters[i].Path = path
|
||||||
|
}
|
||||||
|
|
||||||
cmds, err := eng.Commands(spec)
|
cmds, err := eng.Commands(spec)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1410,215 +1041,6 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) {
|
|||||||
return &http.Client{Transport: &http2.Transport{TLSClientConfig: tc}}, nil
|
return &http.Client{Transport: &http2.Transport{TLSClientConfig: tc}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// engines is what this node can run: what the agent installed and saw run,
|
|
||||||
// and what the configuration names and answers. Nothing merely found on the
|
|
||||||
// machine — a llama-server on PATH is somebody's, not necessarily one that
|
|
||||||
// works, and placing on it is how a replica fails on a node nobody watches.
|
|
||||||
func (a *Agent) engines(ctx context.Context) map[string]string {
|
|
||||||
a.engineOnce.Do(func() {
|
|
||||||
vers := a.reg.ProbeConfigured(ctx)
|
|
||||||
a.mu.Lock()
|
|
||||||
a.configuredVers, a.lastEngineProbe = vers, time.Now()
|
|
||||||
a.mu.Unlock()
|
|
||||||
if len(vers) > 0 {
|
|
||||||
a.log.Info("engines named in the configuration",
|
|
||||||
"engines", engines.ProbedNames(vers))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
a.mu.Lock()
|
|
||||||
defer a.mu.Unlock()
|
|
||||||
vers := a.engineReportLocked()
|
|
||||||
// Whatever registration sends is what the control plane now knows, so a
|
|
||||||
// heartbeat repeats it only once it differs.
|
|
||||||
a.enginesSent = enginesFingerprint(vers)
|
|
||||||
return vers
|
|
||||||
}
|
|
||||||
|
|
||||||
// engineReportLocked merges the two sources. Installed wins: it is the one
|
|
||||||
// that was seen to run. Called with mu held.
|
|
||||||
func (a *Agent) engineReportLocked() map[string]string {
|
|
||||||
out := maps.Clone(a.configuredVers)
|
|
||||||
if out == nil {
|
|
||||||
out = map[string]string{}
|
|
||||||
}
|
|
||||||
maps.Copy(out, a.installedVers)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// provisionEvery is how often installed engines are checked again. A library
|
|
||||||
// removed by an unrelated upgrade is found here rather than by the next
|
|
||||||
// replica to start.
|
|
||||||
const provisionEvery = 10 * time.Minute
|
|
||||||
|
|
||||||
// setProvision takes the control plane's list of wanted engines.
|
|
||||||
func (a *Agent) setProvision(p *llmgwv1.Provision) {
|
|
||||||
wants := []provision.Want{}
|
|
||||||
for _, e := range p.GetEngines() {
|
|
||||||
wants = append(wants, provision.Want{Engine: e.GetEngine(), Version: e.GetVersion()})
|
|
||||||
}
|
|
||||||
a.mu.Lock()
|
|
||||||
a.provWants = wants
|
|
||||||
a.mu.Unlock()
|
|
||||||
select {
|
|
||||||
case a.provKick <- struct{}{}:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// provisionLoop installs and re-checks the wanted engines, one pass at a time.
|
|
||||||
func (a *Agent) provisionLoop(ctx context.Context) {
|
|
||||||
// After a failure the next pass comes sooner, and backs off: a lock held
|
|
||||||
// by a new machine's own updates clears in a minute or two, while ten
|
|
||||||
// minutes' wait left a node idle for no reason.
|
|
||||||
next := provisionEvery
|
|
||||||
retry := time.Minute
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-a.provKick:
|
|
||||||
case <-time.After(next):
|
|
||||||
}
|
|
||||||
a.mu.Lock()
|
|
||||||
wants := slices.Clone(a.provWants)
|
|
||||||
managed := a.provWants != nil
|
|
||||||
a.mu.Unlock()
|
|
||||||
if !managed {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
statuses := a.prov.Converge(ctx, wants)
|
|
||||||
next = provisionEvery
|
|
||||||
for _, st := range statuses {
|
|
||||||
if st.State == provision.Failed {
|
|
||||||
next, retry = retry, min(retry*2, provisionEvery)
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if next == provisionEvery {
|
|
||||||
retry = time.Minute
|
|
||||||
}
|
|
||||||
installed := map[string]string{}
|
|
||||||
keep := map[string]bool{}
|
|
||||||
for _, st := range statuses {
|
|
||||||
keep[st.Engine] = true
|
|
||||||
if st.State == provision.Ready {
|
|
||||||
a.reg.UseInstalled(st.Engine, st.Binary)
|
|
||||||
installed[st.Engine] = st.Version
|
|
||||||
}
|
|
||||||
}
|
|
||||||
a.mu.Lock()
|
|
||||||
if enginesFingerprint(installed) != enginesFingerprint(a.installedVers) {
|
|
||||||
a.log.Info("engines installed and running", "engines", engines.ProbedNames(installed))
|
|
||||||
}
|
|
||||||
a.installedVers = installed
|
|
||||||
// An engine no longer asked for is no longer reported on.
|
|
||||||
for e := range a.provStatus {
|
|
||||||
if !keep[e] {
|
|
||||||
delete(a.provStatus, e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
a.mu.Unlock()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// provisionStatus records a change in one engine's installation.
|
|
||||||
func (a *Agent) provisionStatus(st provision.Status) {
|
|
||||||
a.mu.Lock()
|
|
||||||
prev, had := a.provStatus[st.Engine]
|
|
||||||
a.provStatus[st.Engine] = st
|
|
||||||
a.mu.Unlock()
|
|
||||||
if !had || prev.State != st.State || prev.Detail != st.Detail {
|
|
||||||
lvl := slog.LevelInfo
|
|
||||||
if st.State == provision.Failed || st.State == provision.Unsupported {
|
|
||||||
lvl = slog.LevelWarn
|
|
||||||
}
|
|
||||||
a.log.Log(context.Background(), lvl, "engine installation",
|
|
||||||
"engine", st.Engine, "version", st.Version, "variant", st.Variant,
|
|
||||||
"state", st.State, "detail", st.Detail)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func provisionFingerprint(m map[string]provision.Status) string {
|
|
||||||
var b strings.Builder
|
|
||||||
for _, k := range slices.Sorted(maps.Keys(m)) {
|
|
||||||
st := m[k]
|
|
||||||
fmt.Fprintf(&b, "%s|%s|%s|%s|%s\n", st.Engine, st.Version, st.Variant, st.State, st.Detail)
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// engineReprobe is how often the installed engines are looked for again.
|
|
||||||
//
|
|
||||||
// Minutes, not seconds: an engine is installed by somebody at a keyboard, so a
|
|
||||||
// minute's delay in noticing is invisible, while SGLang's probe starts a
|
|
||||||
// Python interpreter that imports it.
|
|
||||||
const engineReprobe = time.Minute
|
|
||||||
|
|
||||||
// reprobeEngines looks for installed engines again when it is time, off the
|
|
||||||
// heartbeat goroutine so a slow probe never delays a heartbeat.
|
|
||||||
func (a *Agent) reprobeEngines() {
|
|
||||||
a.mu.Lock()
|
|
||||||
due := !a.probingEngines && !a.lastEngineProbe.IsZero() &&
|
|
||||||
time.Since(a.lastEngineProbe) >= engineReprobe
|
|
||||||
if due {
|
|
||||||
a.probingEngines = true
|
|
||||||
}
|
|
||||||
a.mu.Unlock()
|
|
||||||
if !due {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
go func() {
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
vers := a.reg.ProbeConfigured(ctx)
|
|
||||||
|
|
||||||
a.mu.Lock()
|
|
||||||
changed := enginesFingerprint(vers) != enginesFingerprint(a.configuredVers)
|
|
||||||
a.configuredVers, a.lastEngineProbe, a.probingEngines = vers, time.Now(), false
|
|
||||||
a.mu.Unlock()
|
|
||||||
if changed {
|
|
||||||
a.log.Info("engines named in the configuration changed",
|
|
||||||
"engines", engines.ProbedNames(vers))
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
|
|
||||||
func enginesFingerprint(vers map[string]string) string {
|
|
||||||
var b strings.Builder
|
|
||||||
for _, k := range slices.Sorted(maps.Keys(vers)) {
|
|
||||||
fmt.Fprintf(&b, "%s=%s\n", k, vers[k])
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// logsReply answers a request for one of this node's logs.
|
|
||||||
func (a *Agent) logsReply(req *llmgwv1.LogsRequest) *llmgwv1.LogsReply {
|
|
||||||
n := int(req.GetLines())
|
|
||||||
if n <= 0 {
|
|
||||||
n = 200
|
|
||||||
}
|
|
||||||
n = min(n, agentLogLines)
|
|
||||||
shards := a.sup.IDs()
|
|
||||||
slices.Sort(shards)
|
|
||||||
out := &llmgwv1.LogsReply{
|
|
||||||
RequestId: req.GetRequestId(),
|
|
||||||
Sources: append([]string{"agent"}, shards...),
|
|
||||||
}
|
|
||||||
switch src := req.GetSource(); src {
|
|
||||||
case "", "agent":
|
|
||||||
out.Lines = a.logRing.tail(n)
|
|
||||||
default:
|
|
||||||
sh, err := a.sup.Get(src)
|
|
||||||
if err != nil {
|
|
||||||
out.Error = "no engine " + src + " runs on this node"
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
out.Lines = sh.Logs(n)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// localizeSpec fills in what only this node knows.
|
// localizeSpec fills in what only this node knows.
|
||||||
//
|
//
|
||||||
// The interface NCCL bootstraps over is this node's, not the cluster's: the
|
// The interface NCCL bootstraps over is this node's, not the cluster's: the
|
||||||
@@ -1669,208 +1091,3 @@ func ResolveManagement(c Config) (addr, iface string) {
|
|||||||
}
|
}
|
||||||
return addr, iface
|
return addr, iface
|
||||||
}
|
}
|
||||||
|
|
||||||
func smKey(hca string, port int) string { return fmt.Sprintf("%s/%d", hca, port) }
|
|
||||||
|
|
||||||
// scanFabric is discovery.ScanFabric, replaced in tests.
|
|
||||||
var scanFabric = discovery.ScanFabric
|
|
||||||
|
|
||||||
// querySMs is discovery.QuerySMs, replaced in tests.
|
|
||||||
var querySMs = func(hca string, port int) ([]discovery.SMRecord, error) {
|
|
||||||
return discovery.QuerySMs(context.Background(), hca, port)
|
|
||||||
}
|
|
||||||
|
|
||||||
func splitSMKey(k string) (string, int) {
|
|
||||||
i := strings.LastIndexByte(k, '/')
|
|
||||||
port, _ := strconv.Atoi(k[i+1:])
|
|
||||||
return k[:i], port
|
|
||||||
}
|
|
||||||
|
|
||||||
// querySM is discovery.QuerySM, replaced in tests.
|
|
||||||
var querySM = func(hca string, port int) (*discovery.SMInfo, error) {
|
|
||||||
return discovery.QuerySM(context.Background(), hca, port)
|
|
||||||
}
|
|
||||||
|
|
||||||
// attachSM puts on each active InfiniBand port the subnet manager it sees,
|
|
||||||
// asking again once a minute and otherwise reusing the last answer.
|
|
||||||
func (a *Agent) attachSM(hcas []discovery.HCA) {
|
|
||||||
a.mu.Lock()
|
|
||||||
stale := time.Since(a.smAsked) > time.Minute || a.sm == nil
|
|
||||||
a.mu.Unlock()
|
|
||||||
if stale {
|
|
||||||
sm, errs := map[string]*discovery.SMInfo{}, map[string]string{}
|
|
||||||
_ = discovery.AttachSM(hcas, func(hca string, port int) (*discovery.SMInfo, error) {
|
|
||||||
got, err := querySM(hca, port)
|
|
||||||
if err != nil {
|
|
||||||
errs[smKey(hca, port)] = err.Error()
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
sm[smKey(hca, port)] = got
|
|
||||||
return got, nil
|
|
||||||
})
|
|
||||||
// Every subnet manager on each fabric, through one port per master:
|
|
||||||
// the same list through every port of one fabric is one query too
|
|
||||||
// many for each.
|
|
||||||
sms, smsErr := map[string][]discovery.SMRecord{}, map[string]string{}
|
|
||||||
asked := map[string]bool{}
|
|
||||||
for k, v := range sm {
|
|
||||||
if asked[v.GUID] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
asked[v.GUID] = true
|
|
||||||
hca, port := splitSMKey(k)
|
|
||||||
if recs, err := querySMs(hca, port); err != nil {
|
|
||||||
smsErr[k] = err.Error()
|
|
||||||
} else {
|
|
||||||
sms[k] = recs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
a.mu.Lock()
|
|
||||||
for k, v := range sm {
|
|
||||||
if old := a.sm[k]; old != nil && old.GUID != v.GUID {
|
|
||||||
a.log.Warn("the master subnet manager changed", "port", k, "from", old.GUID, "to", v.GUID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
a.sm, a.smErr, a.smAsked = sm, errs, time.Now()
|
|
||||||
a.sms, a.smsErr = sms, smsErr
|
|
||||||
a.mu.Unlock()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.mu.Lock()
|
|
||||||
defer a.mu.Unlock()
|
|
||||||
for i := range hcas {
|
|
||||||
for j := range hcas[i].Ports {
|
|
||||||
if sm := a.sm[smKey(hcas[i].Name, hcas[i].Ports[j].Num)]; sm != nil && hcas[i].Ports[j].Active() {
|
|
||||||
hcas[i].Ports[j].SM = sm
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ensureDiags installs infiniband-diags on a node with adapters and without
|
|
||||||
// it. sminfo is what tells which physical fabric a port is on; without it,
|
|
||||||
// two separate fabrics given one name cannot be caught. Installed the way
|
|
||||||
// engine dependencies are, from the machine's own package sources.
|
|
||||||
func (a *Agent) ensureDiags(ctx context.Context) {
|
|
||||||
a.mu.Lock()
|
|
||||||
hasIB := len(a.inv.HCAs) > 0
|
|
||||||
a.mu.Unlock()
|
|
||||||
if !hasIB || a.prov == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if _, err := exec.LookPath("sminfo"); err == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.log.Info("installing infiniband-diags, to tell which fabric each port is on")
|
|
||||||
pkgs := map[string][]string{"deb": {"infiniband-diags"}, "rpm": {"infiniband-diags"}}
|
|
||||||
if err := a.prov.InstallPackages(ctx, pkgs); err != nil {
|
|
||||||
a.log.Warn("could not install infiniband-diags; fabrics cannot be told apart by their subnet manager",
|
|
||||||
"err", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// FeatureFabricTest is the agent answering FabricTestRequest.
|
|
||||||
const FeatureFabricTest = "fabric-test"
|
|
||||||
|
|
||||||
// runWriteBW is discovery.RunWriteBW, and haveWriteBW whether it is
|
|
||||||
// installed; both replaced in tests.
|
|
||||||
var (
|
|
||||||
runWriteBW = discovery.RunWriteBW
|
|
||||||
haveWriteBW = func() bool { _, err := exec.LookPath("ib_write_bw"); return err == nil }
|
|
||||||
)
|
|
||||||
|
|
||||||
// fabricTest runs this node's side of a bandwidth test with another.
|
|
||||||
func (a *Agent) fabricTest(ctx context.Context, req *llmgwv1.FabricTestRequest) *llmgwv1.FabricTestReply {
|
|
||||||
start := time.Now()
|
|
||||||
reply := &llmgwv1.FabricTestReply{RequestId: req.GetRequestId()}
|
|
||||||
defer func() { reply.TookMs = time.Since(start).Milliseconds() }()
|
|
||||||
|
|
||||||
w := discovery.WriteBW{HCA: req.GetHca(), Port: int(req.GetPort()), GIDIndex: -1,
|
|
||||||
TCPPort: int(req.GetTcpPort()), Seconds: int(req.GetSeconds())}
|
|
||||||
if req.GetRole() == "client" {
|
|
||||||
w.Peer = req.GetPeer()
|
|
||||||
}
|
|
||||||
a.mu.Lock()
|
|
||||||
found := false
|
|
||||||
for _, h := range a.inv.HCAs {
|
|
||||||
for _, p := range h.Ports {
|
|
||||||
if h.Name == w.HCA && p.Num == w.Port {
|
|
||||||
found = true
|
|
||||||
// RoCE needs its v2 GID named, as NCCL does; InfiniBand uses the LID.
|
|
||||||
if p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID >= 0 {
|
|
||||||
w.GIDIndex = p.RoCEv2GID
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
a.mu.Unlock()
|
|
||||||
if !found {
|
|
||||||
reply.Error = fmt.Sprintf("this node has no port %s/%d", w.HCA, w.Port)
|
|
||||||
return reply
|
|
||||||
}
|
|
||||||
if !haveWriteBW() && a.prov != nil {
|
|
||||||
a.log.Info("installing perftest, for a bandwidth test with another node")
|
|
||||||
pkgs := map[string][]string{"deb": {"perftest"}, "rpm": {"perftest"}}
|
|
||||||
if err := a.prov.InstallPackages(ctx, pkgs); err != nil {
|
|
||||||
reply.Error = "installing perftest: " + err.Error()
|
|
||||||
return reply
|
|
||||||
}
|
|
||||||
}
|
|
||||||
gbps, out, err := runWriteBW(ctx, w)
|
|
||||||
reply.Gbps = gbps
|
|
||||||
reply.Output = tailLines(out, 12)
|
|
||||||
if err != nil {
|
|
||||||
reply.Error = err.Error()
|
|
||||||
}
|
|
||||||
return reply
|
|
||||||
}
|
|
||||||
|
|
||||||
func tailLines(s string, n int) string {
|
|
||||||
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
|
|
||||||
if len(lines) > n {
|
|
||||||
lines = lines[len(lines)-n:]
|
|
||||||
}
|
|
||||||
return strings.Join(lines, "\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
// discoverGPUFabric and discoverIMEX are replaced in tests.
|
|
||||||
var (
|
|
||||||
discoverGPUFabric = discovery.DiscoverGPUFabric
|
|
||||||
discoverIMEX = discovery.DiscoverIMEX
|
|
||||||
)
|
|
||||||
|
|
||||||
// sampleNVLink reads, once a minute, which NVLink domain each GPU is joined
|
|
||||||
// to and whether IMEX is up. nvidia-smi -q takes a second or two, too long
|
|
||||||
// for every heartbeat, and a fabric that was joined an hour ago rarely
|
|
||||||
// changes in five seconds — but when a fabric manager restarts it does, for
|
|
||||||
// every card at once.
|
|
||||||
func (a *Agent) sampleNVLink(hb *llmgwv1.Heartbeat) {
|
|
||||||
a.mu.Lock()
|
|
||||||
gpus := slices.Clone(a.inv.GPUs)
|
|
||||||
due := time.Since(a.nvlAsked) > time.Minute
|
|
||||||
if due {
|
|
||||||
a.nvlAsked = time.Now()
|
|
||||||
}
|
|
||||||
a.mu.Unlock()
|
|
||||||
if len(gpus) == 0 || !due {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
fab, err := discoverGPUFabric(ctx, a.cfg.Discovery.NvidiaSMI)
|
|
||||||
if err != nil || len(fab) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
discovery.ApplyFabric(gpus, fab)
|
|
||||||
for _, g := range gpus {
|
|
||||||
if g.Fabric == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
hb.GpuFabric = append(hb.GpuFabric, &llmgwv1.GPUFabricState{
|
|
||||||
Index: int32(g.Index), State: g.Fabric.State, Status: g.Fabric.Status,
|
|
||||||
ClusterUuid: g.Fabric.ClusterUUID, CliqueId: g.Fabric.CliqueID, Bandwidth: g.Fabric.Bandwidth,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
m := discoverIMEX(ctx)
|
|
||||||
hb.Imex = &llmgwv1.IMEX{State: m.State, Channels: int32(m.Channels)}
|
|
||||||
}
|
|
||||||
@@ -400,11 +400,16 @@ func TestHeartbeatKeepsNodeAlive(t *testing.T) {
|
|||||||
n, _ := h.inv.Get("n1")
|
n, _ := h.inv.Get("n1")
|
||||||
first := n.LastSeen
|
first := n.LastSeen
|
||||||
|
|
||||||
time.Sleep(200 * time.Millisecond)
|
// Waited for rather than slept through: four heartbeat intervals was not
|
||||||
n, _ = h.inv.Get("n1")
|
// always enough on a loaded runner for one to have landed.
|
||||||
if !n.LastSeen.After(first) {
|
deadline := time.Now().Add(3 * time.Second)
|
||||||
t.Error("heartbeats are not refreshing LastSeen; the node would be declared lost")
|
for time.Now().Before(deadline) {
|
||||||
|
if n, _ = h.inv.Get("n1"); n.LastSeen.After(first) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
}
|
}
|
||||||
|
t.Error("heartbeats are not refreshing LastSeen; the node would be declared lost")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStartAndStopShardEndToEnd(t *testing.T) {
|
func TestStartAndStopShardEndToEnd(t *testing.T) {
|
||||||
|
|||||||
@@ -0,0 +1,286 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Config configures the agent.
|
||||||
|
type Config struct {
|
||||||
|
// NodeID defaults to the hostname. It must equal the common name of the
|
||||||
|
// agent's certificate, so set it only where the two differ.
|
||||||
|
NodeID string `koanf:"node_id"`
|
||||||
|
// Server is the control plane, and Servers is the same thing when there is
|
||||||
|
// more than one. Both are accepted: every configuration written before
|
||||||
|
// there could be a second control plane names one address, and none of
|
||||||
|
// them should have to change.
|
||||||
|
//
|
||||||
|
// The order is tried in order, and a refusal moves to the next. Only the
|
||||||
|
// leader accepts an agent, so a standby is skipped in the time it takes to
|
||||||
|
// be told so — which is what makes failover a reconnect rather than an
|
||||||
|
// edit to every node's configuration.
|
||||||
|
Server string `koanf:"server"`
|
||||||
|
Servers []string `koanf:"servers"`
|
||||||
|
// Hostname defaults to the machine's.
|
||||||
|
Hostname string `koanf:"hostname"`
|
||||||
|
// MgmtAddr is the address engines are reached on. It defaults to this
|
||||||
|
// machine's address on the route to the control plane, which is the one
|
||||||
|
// the control plane and the router can reach.
|
||||||
|
MgmtAddr string `koanf:"mgmt_addr"`
|
||||||
|
// Labels are free-form facts about a node for a profile's node_selector.
|
||||||
|
// Nothing needs them: memory, GPUs, engines and disk are measured, and a
|
||||||
|
// label restating one of those goes stale the day the machine changes.
|
||||||
|
Labels map[string]string `koanf:"labels"`
|
||||||
|
|
||||||
|
// MgmtInterface is the ethernet interface NCCL bootstraps over. It
|
||||||
|
// defaults to the interface holding MgmtAddr, then eth0.
|
||||||
|
MgmtInterface string `koanf:"mgmt_interface"`
|
||||||
|
// Fabrics labels this node's HCAs. It is explicit on purpose: OpenSM's
|
||||||
|
// default subnet prefix is the same on every fabric, so nothing can be
|
||||||
|
// inferred from the wire (plan §5.2).
|
||||||
|
Fabrics []inventory.FabricAttachment `koanf:"fabrics"`
|
||||||
|
|
||||||
|
HeartbeatInterval time.Duration `koanf:"heartbeat_interval"`
|
||||||
|
// ReconnectBase is the first reconnect delay; it backs off from there.
|
||||||
|
ReconnectBase time.Duration `koanf:"reconnect_base"`
|
||||||
|
ReconnectMax time.Duration `koanf:"reconnect_max"`
|
||||||
|
|
||||||
|
// AdoptReadyTimeout bounds how long an adopted engine is given to answer a
|
||||||
|
// health check before the agent stops waiting on it.
|
||||||
|
//
|
||||||
|
// Shorter than a startup timeout on purpose: this engine is already
|
||||||
|
// running, so it is not loading a model, it is either serving or it is not.
|
||||||
|
AdoptReadyTimeout time.Duration `koanf:"adopt_ready_timeout"`
|
||||||
|
|
||||||
|
// TLS authenticates both ends of the control-plane stream. Leaving it unset
|
||||||
|
// keeps the plaintext behaviour, which only belongs on a trusted network.
|
||||||
|
TLS tlsconf.Config `koanf:"tls"`
|
||||||
|
|
||||||
|
// ModelDirs are the directories this node keeps model weights in, beyond
|
||||||
|
// WeightsCache, which is always looked in. Storage differs at every installation,
|
||||||
|
// so llmgw observes it rather than owning it: a Lustre mount, an image
|
||||||
|
// exported over the fabric, local NVMe and a copy made by hand all look
|
||||||
|
// identical from here, and placement only needs to know whether the file
|
||||||
|
// is present.
|
||||||
|
ModelDirs []string `koanf:"model_dirs"`
|
||||||
|
|
||||||
|
// EnginesDir is where the agent installs engines, one directory per
|
||||||
|
// build. Defaults to /opt/llmgw/engines.
|
||||||
|
EnginesDir string `koanf:"engines_dir"`
|
||||||
|
|
||||||
|
// WeightsCache is where a remote source is downloaded to. Defaults to the
|
||||||
|
// first entry of ModelDirs, because a node that says where its models live
|
||||||
|
// has already answered this question, and otherwise to
|
||||||
|
// /var/lib/llmgw-weights.
|
||||||
|
//
|
||||||
|
// Fetching is the simple arrangement: every node keeps its own copy. The
|
||||||
|
// exotic ones need no code — weights on a shared filer or an image exported
|
||||||
|
// over the fabric are simply present, and present means nothing is fetched.
|
||||||
|
WeightsCache string `koanf:"weights_cache"`
|
||||||
|
|
||||||
|
Discovery discovery.Options `koanf:"discovery"`
|
||||||
|
Engines engines.Config `koanf:"engines"`
|
||||||
|
Superv supervisor.Config `koanf:"supervisor"`
|
||||||
|
|
||||||
|
Version string
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultConfig returns sensible defaults.
|
||||||
|
func DefaultConfig() Config {
|
||||||
|
// No MgmtInterface: normalize finds the one holding the management
|
||||||
|
// address, and falls back to eth0 only when it cannot. Defaulted here, it
|
||||||
|
// was never found — every node said eth0 unless told otherwise, and a node
|
||||||
|
// on ens3 or bond0 had NCCL bootstrap over an interface it does not have.
|
||||||
|
return Config{
|
||||||
|
HeartbeatInterval: 5 * time.Second,
|
||||||
|
ReconnectBase: time.Second,
|
||||||
|
ReconnectMax: 30 * time.Second,
|
||||||
|
AdoptReadyTimeout: 60 * time.Second,
|
||||||
|
Superv: supervisor.DefaultConfig(),
|
||||||
|
// NVML gives NVLink topology, so prefer it when this build has it.
|
||||||
|
Discovery: discovery.Options{PreferNVML: true},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Config) normalize() error {
|
||||||
|
d := DefaultConfig()
|
||||||
|
if err := c.TLS.Validate(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := c.normalizeServerURL(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Everything below is found rather than written. A file that has to say
|
||||||
|
// which machine it is on, and at what address, is a file that differs on
|
||||||
|
// every node — and a copy taken from one node to the next registers the
|
||||||
|
// second as the first.
|
||||||
|
host, _ := os.Hostname()
|
||||||
|
if c.NodeID == "" {
|
||||||
|
c.NodeID = host
|
||||||
|
}
|
||||||
|
if c.NodeID == "" {
|
||||||
|
return errors.New("node_id is not set and the hostname could not be read")
|
||||||
|
}
|
||||||
|
if c.Hostname == "" {
|
||||||
|
c.Hostname = host
|
||||||
|
}
|
||||||
|
if c.MgmtAddr == "" {
|
||||||
|
c.MgmtAddr = addrToward(c.Server)
|
||||||
|
}
|
||||||
|
if c.MgmtInterface == "" {
|
||||||
|
c.MgmtInterface = interfaceOf(c.MgmtAddr)
|
||||||
|
}
|
||||||
|
if c.MgmtInterface == "" {
|
||||||
|
c.MgmtInterface = "eth0"
|
||||||
|
}
|
||||||
|
// The directory downloads land in is always one that is looked in. When
|
||||||
|
// they were two settings, a node could fill its disk with fetched weights
|
||||||
|
// that placement never saw — which is what every node here was doing.
|
||||||
|
if c.WeightsCache == "" && len(c.ModelDirs) > 0 {
|
||||||
|
c.WeightsCache = c.ModelDirs[0]
|
||||||
|
}
|
||||||
|
if c.WeightsCache == "" {
|
||||||
|
c.WeightsCache = DefaultWeightsCache
|
||||||
|
}
|
||||||
|
if !slices.Contains(c.ModelDirs, c.WeightsCache) {
|
||||||
|
c.ModelDirs = append(c.ModelDirs, c.WeightsCache)
|
||||||
|
}
|
||||||
|
if c.HeartbeatInterval <= 0 {
|
||||||
|
c.HeartbeatInterval = d.HeartbeatInterval
|
||||||
|
}
|
||||||
|
if c.ReconnectBase <= 0 {
|
||||||
|
c.ReconnectBase = d.ReconnectBase
|
||||||
|
}
|
||||||
|
if c.AdoptReadyTimeout <= 0 {
|
||||||
|
c.AdoptReadyTimeout = d.AdoptReadyTimeout
|
||||||
|
}
|
||||||
|
if c.ReconnectMax < c.ReconnectBase {
|
||||||
|
c.ReconnectMax = max(c.ReconnectBase, d.ReconnectMax)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizeServerURL makes the configured address a URL connect can use, and
|
||||||
|
// refuses the combination that would quietly undo the certificates: a scheme
|
||||||
|
// that says plaintext while TLS is configured. Silently connecting anyway would
|
||||||
|
// mean a cluster believing it has mutual authentication when it has none.
|
||||||
|
func (c *Config) normalizeServerURL() error {
|
||||||
|
// One list, however it was written. Server first when both are given,
|
||||||
|
// because an operator adding a standby to an existing file expects the
|
||||||
|
// one already there to stay preferred.
|
||||||
|
all := make([]string, 0, 1+len(c.Servers))
|
||||||
|
if c.Server != "" {
|
||||||
|
all = append(all, c.Server)
|
||||||
|
}
|
||||||
|
all = append(all, c.Servers...)
|
||||||
|
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
out := make([]string, 0, len(all))
|
||||||
|
for _, s := range all {
|
||||||
|
fixed, err := normalizeOne(s, c.TLS.Enabled())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// A duplicate would be tried twice in a row on failover, which reads
|
||||||
|
// as the rotation not working.
|
||||||
|
if _, dup := seen[fixed]; dup {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[fixed] = struct{}{}
|
||||||
|
out = append(out, fixed)
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
return errors.New("server is required")
|
||||||
|
}
|
||||||
|
c.Servers = out
|
||||||
|
c.Server = out[0]
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultWeightsCache is where fetched weights go when nothing says otherwise.
|
||||||
|
const DefaultWeightsCache = "/var/lib/llmgw-weights"
|
||||||
|
|
||||||
|
// addrToward is this machine's address on the route to the control plane.
|
||||||
|
//
|
||||||
|
// The address engines are reached on is whichever one the control plane can
|
||||||
|
// reach, and the kernel already knows which that is: a UDP "connection" picks
|
||||||
|
// the source address for the route without sending a packet. Asking the
|
||||||
|
// interfaces instead would have to guess among docker bridges, VPNs and
|
||||||
|
// link-local addresses.
|
||||||
|
func addrToward(server string) string {
|
||||||
|
u, err := url.Parse(server)
|
||||||
|
if err != nil || u.Host == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
host := u.Hostname()
|
||||||
|
port := u.Port()
|
||||||
|
if port == "" {
|
||||||
|
port = "443"
|
||||||
|
}
|
||||||
|
conn, err := net.Dial("udp", net.JoinHostPort(host, port))
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
if a, ok := conn.LocalAddr().(*net.UDPAddr); ok {
|
||||||
|
return a.IP.String()
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// interfaceOf names the interface that holds an address, or "" when none does.
|
||||||
|
func interfaceOf(addr string) string {
|
||||||
|
ip := net.ParseIP(addr)
|
||||||
|
if ip == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
ifaces, err := net.Interfaces()
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
for _, ifc := range ifaces {
|
||||||
|
addrs, err := ifc.Addrs()
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, a := range addrs {
|
||||||
|
if n, ok := a.(*net.IPNet); ok && n.IP.Equal(ip) {
|
||||||
|
return ifc.Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizeOne applies the scheme rules to a single address.
|
||||||
|
func normalizeOne(addr string, secure bool) (string, error) {
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(addr, "https://"):
|
||||||
|
if !secure {
|
||||||
|
return "", fmt.Errorf("%s is an https:// address but no tls certificates are configured", addr)
|
||||||
|
}
|
||||||
|
case strings.HasPrefix(addr, "http://"):
|
||||||
|
if secure {
|
||||||
|
return "", fmt.Errorf("tls is configured but %s is an http:// address; use https://", addr)
|
||||||
|
}
|
||||||
|
case strings.Contains(addr, "://"):
|
||||||
|
return "", fmt.Errorf("server %q must be an http:// or https:// address", addr)
|
||||||
|
case secure:
|
||||||
|
return "https://" + addr, nil
|
||||||
|
default:
|
||||||
|
return "http://" + addr, nil
|
||||||
|
}
|
||||||
|
return addr, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,242 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Sensors is what a node's own hardware says about heat and power: the CPUs,
|
||||||
|
// the board where it has an inlet sensor, and each GPU. A zero is not
|
||||||
|
// measured, never a reading of zero — a VM has none of these, and must not
|
||||||
|
// look like a machine running at 0 °C on no power.
|
||||||
|
type Sensors struct {
|
||||||
|
// CPUTempC is the hottest CPU sensor: the package, or the hottest core
|
||||||
|
// where no package reading exists.
|
||||||
|
CPUTempC float64
|
||||||
|
// CPUPowerW is the CPU packages' draw, from RAPL: the change in their
|
||||||
|
// energy counters over the time between two samples.
|
||||||
|
CPUPowerW float64
|
||||||
|
// InletTempC is the air coming in, where the board has a sensor it calls
|
||||||
|
// that. Most do not, and a CPU's temperature is not a substitute: it
|
||||||
|
// follows the load, not the room.
|
||||||
|
InletTempC float64
|
||||||
|
GPUs []GPUSensor
|
||||||
|
}
|
||||||
|
|
||||||
|
// GPUSensor is one card's heat, power and whether it is slowing itself down.
|
||||||
|
type GPUSensor struct {
|
||||||
|
Index int
|
||||||
|
TempC float64
|
||||||
|
MemoryTempC float64
|
||||||
|
PowerW float64
|
||||||
|
PowerLimitW float64
|
||||||
|
// Throttle names what is holding the card's clocks down now:
|
||||||
|
// sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake. Empty
|
||||||
|
// is running free. Idle and application clock settings are left out;
|
||||||
|
// they are not the card being held back.
|
||||||
|
Throttle []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// sensorEvery bounds how often nvidia-smi is run. Heartbeats come more often
|
||||||
|
// than temperatures change, and each run costs a few hundred milliseconds.
|
||||||
|
const sensorEvery = 10 * time.Second
|
||||||
|
|
||||||
|
// SensorSampler reads Sensors. The first sample has no CPU power, which needs
|
||||||
|
// two readings of the energy counters.
|
||||||
|
type SensorSampler struct {
|
||||||
|
sys string // /sys, or a test's copy of it
|
||||||
|
smi func(ctx context.Context, fields string) ([]byte, error)
|
||||||
|
now func() time.Time
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
last Sensors
|
||||||
|
lastAt time.Time
|
||||||
|
prevEnergy map[string]uint64
|
||||||
|
prevAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSensorSampler reads this machine.
|
||||||
|
func NewSensorSampler() *SensorSampler {
|
||||||
|
return &SensorSampler{sys: "/sys", smi: runSMI, now: time.Now}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runSMI(ctx context.Context, fields string) ([]byte, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
//nolint:gosec // fixed binary, fields from this file
|
||||||
|
return exec.CommandContext(ctx, "nvidia-smi", "--query-gpu="+fields,
|
||||||
|
"--format=csv,noheader,nounits").Output()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sample reads the sensors, or returns the last reading when that is recent.
|
||||||
|
func (s *SensorSampler) Sample(ctx context.Context) Sensors {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
now := s.now()
|
||||||
|
if !s.lastAt.IsZero() && now.Sub(s.lastAt) < sensorEvery {
|
||||||
|
return s.last
|
||||||
|
}
|
||||||
|
out := Sensors{}
|
||||||
|
out.CPUTempC, out.InletTempC = s.hwmon()
|
||||||
|
out.CPUPowerW = s.rapl(now)
|
||||||
|
out.GPUs = s.gpus(ctx)
|
||||||
|
s.last, s.lastAt = out, now
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// hwmon reads the hottest CPU sensor and an inlet sensor, from every hwmon
|
||||||
|
// device the kernel exposes — what lm_sensors reads, without it installed.
|
||||||
|
func (s *SensorSampler) hwmon() (cpu, inlet float64) {
|
||||||
|
devs, _ := filepath.Glob(filepath.Join(s.sys, "class/hwmon/hwmon*"))
|
||||||
|
var pkg, core float64
|
||||||
|
for _, d := range devs {
|
||||||
|
name := readTrim(filepath.Join(d, "name"))
|
||||||
|
inputs, _ := filepath.Glob(filepath.Join(d, "temp*_input"))
|
||||||
|
for _, in := range inputs {
|
||||||
|
milli, err := strconv.ParseFloat(readTrim(in), 64)
|
||||||
|
if err != nil || milli <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c := milli / 1000
|
||||||
|
label := strings.ToLower(readTrim(strings.TrimSuffix(in, "_input") + "_label"))
|
||||||
|
switch {
|
||||||
|
case isCPUSensor(name) && (strings.HasPrefix(label, "package") || label == "tctl" || label == "tdie"):
|
||||||
|
pkg = max(pkg, c)
|
||||||
|
case isCPUSensor(name):
|
||||||
|
core = max(core, c)
|
||||||
|
case label == "cpu":
|
||||||
|
// A board's own reading of the CPU, as Dell's SMM gives.
|
||||||
|
core = max(core, c)
|
||||||
|
case strings.Contains(label, "inlet") || strings.Contains(label, "ambient") ||
|
||||||
|
strings.Contains(label, "intake") || label == "systin":
|
||||||
|
inlet = max(inlet, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if pkg > 0 {
|
||||||
|
return pkg, inlet
|
||||||
|
}
|
||||||
|
return core, inlet
|
||||||
|
}
|
||||||
|
|
||||||
|
func isCPUSensor(name string) bool {
|
||||||
|
switch name {
|
||||||
|
case "coretemp", "k10temp", "zenpower", "cpu_thermal":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// rapl is the CPU packages' power from their energy counters. Package
|
||||||
|
// domains only: the dram and core subdomains are parts of a package, and
|
||||||
|
// adding them would count the same energy twice.
|
||||||
|
func (s *SensorSampler) rapl(now time.Time) float64 {
|
||||||
|
doms, _ := filepath.Glob(filepath.Join(s.sys, "class/powercap/intel-rapl:*"))
|
||||||
|
energy := map[string]uint64{}
|
||||||
|
wrap := map[string]uint64{}
|
||||||
|
for _, d := range doms {
|
||||||
|
if strings.Count(filepath.Base(d), ":") != 1 {
|
||||||
|
continue // intel-rapl:0:0 is a subdomain of intel-rapl:0
|
||||||
|
}
|
||||||
|
e, err := strconv.ParseUint(readTrim(filepath.Join(d, "energy_uj")), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
continue // unreadable without root on newer kernels
|
||||||
|
}
|
||||||
|
energy[d] = e
|
||||||
|
wrap[d], _ = strconv.ParseUint(readTrim(filepath.Join(d, "max_energy_range_uj")), 10, 64)
|
||||||
|
}
|
||||||
|
prev, prevAt := s.prevEnergy, s.prevAt
|
||||||
|
s.prevEnergy, s.prevAt = energy, now
|
||||||
|
dt := now.Sub(prevAt).Seconds()
|
||||||
|
if prev == nil || dt <= 0 || len(energy) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
var joules float64
|
||||||
|
for d, e := range energy {
|
||||||
|
p, ok := prev[d]
|
||||||
|
if !ok {
|
||||||
|
return 0 // a domain appeared: no fair difference this time
|
||||||
|
}
|
||||||
|
if e < p { // the counter wrapped
|
||||||
|
e += wrap[d]
|
||||||
|
}
|
||||||
|
joules += float64(e-p) / 1e6
|
||||||
|
}
|
||||||
|
return joules / dt
|
||||||
|
}
|
||||||
|
|
||||||
|
// The throttle reasons nvidia-smi reports, as bits. Only those that mean the
|
||||||
|
// card is being held back are named.
|
||||||
|
var throttleBits = []struct {
|
||||||
|
bit uint64
|
||||||
|
name string
|
||||||
|
}{
|
||||||
|
{0x4, "sw_power_cap"},
|
||||||
|
{0x8, "hw_slowdown"},
|
||||||
|
{0x20, "sw_thermal"},
|
||||||
|
{0x40, "hw_thermal"},
|
||||||
|
{0x80, "power_brake"},
|
||||||
|
}
|
||||||
|
|
||||||
|
// gpus asks nvidia-smi for each card's heat and power. The memory temperature
|
||||||
|
// is not known to every driver, and a field nvidia-smi does not know fails the
|
||||||
|
// whole query, so it is asked for first and dropped on failure.
|
||||||
|
func (s *SensorSampler) gpus(ctx context.Context) []GPUSensor {
|
||||||
|
withMem := "index,temperature.gpu,power.draw,enforced.power.limit,clocks_throttle_reasons.active,temperature.memory"
|
||||||
|
out, err := s.smi(ctx, withMem)
|
||||||
|
if err != nil {
|
||||||
|
if out, err = s.smi(ctx, strings.TrimSuffix(withMem, ",temperature.memory")); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var gs []GPUSensor
|
||||||
|
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||||
|
f := strings.Split(line, ",")
|
||||||
|
if len(f) < 5 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for i := range f {
|
||||||
|
f[i] = strings.TrimSpace(f[i])
|
||||||
|
}
|
||||||
|
idx, err := strconv.Atoi(f[0])
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
g := GPUSensor{Index: idx, TempC: num(f[1]), PowerW: num(f[2]), PowerLimitW: num(f[3])}
|
||||||
|
if bits, err := strconv.ParseUint(strings.TrimPrefix(f[4], "0x"), 16, 64); err == nil {
|
||||||
|
for _, b := range throttleBits {
|
||||||
|
if bits&b.bit != 0 {
|
||||||
|
g.Throttle = append(g.Throttle, b.name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(f) > 5 {
|
||||||
|
g.MemoryTempC = num(f[5])
|
||||||
|
}
|
||||||
|
gs = append(gs, g)
|
||||||
|
}
|
||||||
|
return gs
|
||||||
|
}
|
||||||
|
|
||||||
|
// num reads a figure nvidia-smi may give as "[N/A]" or "N/A".
|
||||||
|
func num(s string) float64 {
|
||||||
|
v, err := strconv.ParseFloat(s, 64)
|
||||||
|
if err != nil || v < 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
func readTrim(path string) string {
|
||||||
|
b, err := os.ReadFile(path) //nolint:gosec // sysfs
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(string(b))
|
||||||
|
}
|
||||||
@@ -0,0 +1,169 @@
|
|||||||
|
package discovery
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func put(t *testing.T, root, rel, body string) {
|
||||||
|
t.Helper()
|
||||||
|
p := filepath.Join(root, rel)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(p, []byte(body+"\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// dellHost is the layout of the Proxmox host these tests were written
|
||||||
|
// against: Dell's SMM with a CPU and SODIMM sensors, coretemp with a package
|
||||||
|
// and cores, and RAPL with a package and its dram subdomain.
|
||||||
|
func dellHost(t *testing.T) string {
|
||||||
|
root := t.TempDir()
|
||||||
|
put(t, root, "class/hwmon/hwmon0/name", "dell_smm")
|
||||||
|
put(t, root, "class/hwmon/hwmon0/temp1_input", "55000")
|
||||||
|
put(t, root, "class/hwmon/hwmon0/temp1_label", "CPU")
|
||||||
|
put(t, root, "class/hwmon/hwmon0/temp2_input", "47000")
|
||||||
|
put(t, root, "class/hwmon/hwmon0/temp2_label", "SODIMM")
|
||||||
|
put(t, root, "class/hwmon/hwmon1/name", "coretemp")
|
||||||
|
put(t, root, "class/hwmon/hwmon1/temp1_input", "58000")
|
||||||
|
put(t, root, "class/hwmon/hwmon1/temp1_label", "Package id 0")
|
||||||
|
put(t, root, "class/hwmon/hwmon1/temp10_input", "61000")
|
||||||
|
put(t, root, "class/hwmon/hwmon1/temp10_label", "Core 8")
|
||||||
|
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1000000000")
|
||||||
|
put(t, root, "class/powercap/intel-rapl:0/max_energy_range_uj", "262143328850")
|
||||||
|
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "5000000")
|
||||||
|
return root
|
||||||
|
}
|
||||||
|
|
||||||
|
func sampler(sys string, smi func(context.Context, string) ([]byte, error), clock *time.Time) *SensorSampler {
|
||||||
|
return &SensorSampler{sys: sys, smi: smi, now: func() time.Time { return *clock }}
|
||||||
|
}
|
||||||
|
|
||||||
|
func noSMI(context.Context, string) ([]byte, error) { return nil, errors.New("not found") }
|
||||||
|
|
||||||
|
func TestAHostsCPUHeatAndPowerAreReadAsLmSensorsWould(t *testing.T) {
|
||||||
|
root := dellHost(t)
|
||||||
|
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||||
|
s := sampler(root, noSMI, &now)
|
||||||
|
|
||||||
|
first := s.Sample(t.Context())
|
||||||
|
// The package reading, not the hotter core: the package is what the
|
||||||
|
// chip itself reports as its temperature.
|
||||||
|
if first.CPUTempC != 58 {
|
||||||
|
t.Errorf("CPU %.1f °C, expected the package's 58", first.CPUTempC)
|
||||||
|
}
|
||||||
|
if first.CPUPowerW != 0 {
|
||||||
|
t.Error("power from one reading of a counter is invented")
|
||||||
|
}
|
||||||
|
if first.InletTempC != 0 || first.GPUs != nil {
|
||||||
|
t.Errorf("sensors this host does not have were reported: %+v", first)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 10 s later the package has used 650 J, and its dram subdomain some of
|
||||||
|
// that — counted once.
|
||||||
|
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1650000000")
|
||||||
|
put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "99000000")
|
||||||
|
now = now.Add(10 * time.Second)
|
||||||
|
if got := s.Sample(t.Context()).CPUPowerW; got < 64.9 || got > 65.1 {
|
||||||
|
t.Errorf("CPU power %.2f W, expected 65", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestACounterThatWrapsIsNotANegativeDraw(t *testing.T) {
|
||||||
|
root := dellHost(t)
|
||||||
|
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||||
|
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "262143000000")
|
||||||
|
s := sampler(root, noSMI, &now)
|
||||||
|
s.Sample(t.Context())
|
||||||
|
// 328,850 µJ to the wrap, then 499,671,150 past it: 500 J in 10 s.
|
||||||
|
put(t, root, "class/powercap/intel-rapl:0/energy_uj", "499671150")
|
||||||
|
now = now.Add(10 * time.Second)
|
||||||
|
if got := s.Sample(t.Context()).CPUPowerW; got < 49.9 || got > 50.1 {
|
||||||
|
t.Errorf("across a wrap: %.2f W, expected 50", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAVMHasNoSensorsAndSaysNothing(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
put(t, root, "class/thermal/cooling_device0/type", "Processor")
|
||||||
|
now := time.Now()
|
||||||
|
got := sampler(root, noSMI, &now).Sample(t.Context())
|
||||||
|
if got.CPUTempC != 0 || got.CPUPowerW != 0 || got.InletTempC != 0 || got.GPUs != nil {
|
||||||
|
t.Errorf("a machine with no sensors reported %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestABoardsInletSensorIsTheRoomNotTheCPU(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
put(t, root, "class/hwmon/hwmon2/name", "nct6798")
|
||||||
|
put(t, root, "class/hwmon/hwmon2/temp1_input", "24500")
|
||||||
|
put(t, root, "class/hwmon/hwmon2/temp1_label", "SYSTIN")
|
||||||
|
put(t, root, "class/hwmon/hwmon3/name", "k10temp")
|
||||||
|
put(t, root, "class/hwmon/hwmon3/temp1_input", "71250")
|
||||||
|
put(t, root, "class/hwmon/hwmon3/temp1_label", "Tctl")
|
||||||
|
now := time.Now()
|
||||||
|
got := sampler(root, noSMI, &now).Sample(t.Context())
|
||||||
|
if got.InletTempC != 24.5 || got.CPUTempC != 71.25 {
|
||||||
|
t.Errorf("inlet %.2f, CPU %.2f", got.InletTempC, got.CPUTempC)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEachGPUSaysHowHotHowHungryAndWhatHoldsItBack(t *testing.T) {
|
||||||
|
var asked []string
|
||||||
|
smi := func(_ context.Context, fields string) ([]byte, error) {
|
||||||
|
asked = append(asked, fields)
|
||||||
|
return []byte(
|
||||||
|
"0, 71, 612.45, 700.00, 0x0000000000000000, 84\n" +
|
||||||
|
"1, 88, 698.10, 700.00, 0x0000000000000024, 95\n" + // sw_power_cap + sw_thermal
|
||||||
|
"2, 45, [N/A], 700.00, 0x0000000000000001, [N/A]\n"), nil // idle only
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
ss := sampler(t.TempDir(), smi, &now)
|
||||||
|
got := ss.Sample(t.Context())
|
||||||
|
if len(got.GPUs) != 3 {
|
||||||
|
t.Fatalf("%+v", got.GPUs)
|
||||||
|
}
|
||||||
|
g0, g1, g2 := got.GPUs[0], got.GPUs[1], got.GPUs[2]
|
||||||
|
if g0.TempC != 71 || g0.PowerW != 612.45 || g0.PowerLimitW != 700 || g0.MemoryTempC != 84 || g0.Throttle != nil {
|
||||||
|
t.Errorf("card 0 %+v", g0)
|
||||||
|
}
|
||||||
|
if strings.Join(g1.Throttle, ",") != "sw_power_cap,sw_thermal" {
|
||||||
|
t.Errorf("card 1 throttled by %v", g1.Throttle)
|
||||||
|
}
|
||||||
|
if g2.PowerW != 0 || g2.MemoryTempC != 0 || g2.Throttle != nil {
|
||||||
|
t.Errorf("card 2: N/A is not a reading, and idle is not being held back: %+v", g2)
|
||||||
|
}
|
||||||
|
// Within ten seconds the last reading is reused rather than running
|
||||||
|
// nvidia-smi on every heartbeat; after, it is asked again.
|
||||||
|
now = now.Add(5 * time.Second)
|
||||||
|
ss.Sample(t.Context())
|
||||||
|
if len(asked) != 1 {
|
||||||
|
t.Errorf("nvidia-smi was asked %d times within ten seconds", len(asked))
|
||||||
|
}
|
||||||
|
now = now.Add(6 * time.Second)
|
||||||
|
ss.Sample(t.Context())
|
||||||
|
if len(asked) != 2 {
|
||||||
|
t.Errorf("nvidia-smi was asked %d times after eleven seconds, expected 2", len(asked))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestADriverThatDoesNotKnowMemoryTemperatureStillReportsTheRest(t *testing.T) {
|
||||||
|
smi := func(_ context.Context, fields string) ([]byte, error) {
|
||||||
|
if strings.Contains(fields, "temperature.memory") {
|
||||||
|
return nil, errors.New(`Field "temperature.memory" is not a valid field to query.`)
|
||||||
|
}
|
||||||
|
return []byte("0, 66, 250.0, 300.0, 0x0000000000000040\n"), nil
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
got := sampler(t.TempDir(), smi, &now).Sample(t.Context())
|
||||||
|
if len(got.GPUs) != 1 || got.GPUs[0].TempC != 66 || strings.Join(got.GPUs[0].Throttle, ",") != "hw_thermal" {
|
||||||
|
t.Errorf("%+v", got.GPUs)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -40,6 +40,9 @@ type Profile struct {
|
|||||||
ModelPath string
|
ModelPath string
|
||||||
// ServedModelName is the name clients use, which need not match the path.
|
// ServedModelName is the name clients use, which need not match the path.
|
||||||
ServedModelName string
|
ServedModelName string
|
||||||
|
// Adapters are LoRA adapters to load beside the model, in order: an
|
||||||
|
// adapter's index is the id requests apply it by.
|
||||||
|
Adapters []Adapter
|
||||||
}
|
}
|
||||||
|
|
||||||
// ShardSpec is one shard of one replica on this node.
|
// ShardSpec is one shard of one replica on this node.
|
||||||
@@ -70,6 +73,17 @@ type ShardSpec struct {
|
|||||||
HTTPPort int
|
HTTPPort int
|
||||||
Profile Profile
|
Profile Profile
|
||||||
|
|
||||||
|
// APIKey is what the engine demands of every request. Engines listen on
|
||||||
|
// all of a node's addresses; without a key anybody who can reach the port
|
||||||
|
// talks to the model directly, past the gateway's authentication, quotas
|
||||||
|
// and accounting. Given to the engine in its environment rather than on
|
||||||
|
// its command line, which every user on the node can read.
|
||||||
|
APIKey string `json:"api_key,omitempty"`
|
||||||
|
// Threads is the CPU cores placement reserved for a shard with no GPU,
|
||||||
|
// and so what its engine computes on. Zero leaves it to the engine's own
|
||||||
|
// configuration.
|
||||||
|
Threads int `json:"threads,omitempty"`
|
||||||
|
|
||||||
// Pool and Fabric are control plane concepts the agent cannot derive. They
|
// Pool and Fabric are control plane concepts the agent cannot derive. They
|
||||||
// are carried so the agent can report them back after a restart; Fabric
|
// are carried so the agent can report them back after a restart; Fabric
|
||||||
// also says, with an "nvl:" prefix, that the replica is inside a
|
// also says, with an "nvl:" prefix, that the replica is inside a
|
||||||
@@ -249,6 +263,11 @@ func baseEnv(s ShardSpec) map[string]string {
|
|||||||
// Profile overrides win: they are the escape hatch for the per-cluster
|
// Profile overrides win: they are the escape hatch for the per-cluster
|
||||||
// tuning no adapter can guess.
|
// tuning no adapter can guess.
|
||||||
for k, v := range s.Profile.Env {
|
for k, v := range s.Profile.Env {
|
||||||
|
// Checked again here, not only where the model was defined: this is
|
||||||
|
// the process about to run as root, whatever sent the profile.
|
||||||
|
if bad, _ := UnsafeEnv(k); bad {
|
||||||
|
continue
|
||||||
|
}
|
||||||
env[k] = v
|
env[k] = v
|
||||||
}
|
}
|
||||||
return env
|
return env
|
||||||
@@ -357,3 +376,10 @@ func quantizationOf(args []string) string {
|
|||||||
}
|
}
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Adapter is a LoRA adapter an engine loads beside its model. Path is a
|
||||||
|
// source until the agent has fetched it, a local file after.
|
||||||
|
type Adapter struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Path string `json:"path"`
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -21,8 +22,21 @@ type LlamaCppConfig struct {
|
|||||||
// does not say. llama.cpp defaults to one, which would make every second
|
// does not say. llama.cpp defaults to one, which would make every second
|
||||||
// request wait behind the first no matter what the router decided.
|
// request wait behind the first no matter what the router decided.
|
||||||
Parallel int `koanf:"parallel"`
|
Parallel int `koanf:"parallel"`
|
||||||
|
// Threads is how many CPU threads a shard with no GPU computes on. Zero
|
||||||
|
// takes half this machine's CPUs.
|
||||||
|
//
|
||||||
|
// Not all of them, which is llama.cpp's own default: two busy servers
|
||||||
|
// that each spin on every CPU fight for them, and on an 8-vCPU node
|
||||||
|
// that made 7B generation fall from 10 tokens a second to under 1 each.
|
||||||
|
// At half, one server alone is about a third slower and two together
|
||||||
|
// keep 5.5 each. A node that only ever runs one CPU replica does better
|
||||||
|
// with all of them; say so here.
|
||||||
|
Threads int `koanf:"threads"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// numCPU is the CPUs this process may run on; a variable for tests.
|
||||||
|
var numCPU = runtime.NumCPU
|
||||||
|
|
||||||
// LlamaCpp builds llama.cpp launch commands.
|
// LlamaCpp builds llama.cpp launch commands.
|
||||||
//
|
//
|
||||||
// It is the engine that makes a cluster without GPUs useful: llama.cpp runs a
|
// It is the engine that makes a cluster without GPUs useful: llama.cpp runs a
|
||||||
@@ -118,10 +132,35 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
|
|||||||
if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") {
|
if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") {
|
||||||
args = append(args, "--n-gpu-layers", "999")
|
args = append(args, "--n-gpu-layers", "999")
|
||||||
}
|
}
|
||||||
|
if len(s.GPUIndices) == 0 && !hasFlag(p.Args, "--threads", "-t") {
|
||||||
|
// What placement reserved first: it is what the node's other
|
||||||
|
// replicas were placed around.
|
||||||
|
n := s.Threads
|
||||||
|
if n <= 0 {
|
||||||
|
n = l.cfg.Threads
|
||||||
|
}
|
||||||
|
if n <= 0 {
|
||||||
|
n = max(numCPU()/2, 1)
|
||||||
|
}
|
||||||
|
args = append(args, "--threads", strconv.Itoa(n))
|
||||||
|
}
|
||||||
|
// Loaded at start, applied per request: a request that names an adapter
|
||||||
|
// carries "lora": [{"id": i, "scale": s}], and one that names none is the
|
||||||
|
// base model. Each is loaded at scale zero, not with --lora and
|
||||||
|
// --lora-init-without-apply: that leaves an adapter's scale at 1, and
|
||||||
|
// llama-server applies it at that scale to every request with no "lora"
|
||||||
|
// of its own — every request for the model itself.
|
||||||
|
for _, a := range p.Adapters {
|
||||||
|
args = append(args, "--lora-scaled", a.Path+":0")
|
||||||
|
}
|
||||||
args = append(args, p.Args...)
|
args = append(args, p.Args...)
|
||||||
|
|
||||||
|
env := baseEnv(s)
|
||||||
|
if s.APIKey != "" {
|
||||||
|
env["LLAMA_API_KEY"] = s.APIKey
|
||||||
|
}
|
||||||
return []Command{{
|
return []Command{{
|
||||||
Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
|
Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(env),
|
||||||
}}, nil
|
}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -245,3 +245,81 @@ func TestAMissingPathIsLeftToTheEngine(t *testing.T) {
|
|||||||
t.Errorf("expected it passed through, got %q %v", got, err)
|
t.Errorf("expected it passed through, got %q %v", got, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLlamaCppIsGivenItsKeyOutOfSightOfTheProcessList(t *testing.T) {
|
||||||
|
// The command line is readable by every user on the node; the
|
||||||
|
// environment of a root process is not.
|
||||||
|
cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, APIKey: "k1",
|
||||||
|
Profile: Profile{ModelPath: "/m.gguf"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if slices.Contains(cmds[0].Args, "k1") || slices.Contains(cmds[0].Args, "--api-key") {
|
||||||
|
t.Errorf("the key is on the command line: %v", cmds[0].Args)
|
||||||
|
}
|
||||||
|
if !slices.Contains(cmds[0].Env, "LLAMA_API_KEY=k1") {
|
||||||
|
t.Error("llama-server was not given its key")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestACPUShardLeavesHalfTheMachineForAnother(t *testing.T) {
|
||||||
|
prev := numCPU
|
||||||
|
numCPU = func() int { return 8 }
|
||||||
|
t.Cleanup(func() { numCPU = prev })
|
||||||
|
|
||||||
|
threads := func(l *LlamaCpp, s ShardSpec) (string, bool) {
|
||||||
|
cmds, err := l.Commands(s)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return flagValue(cmds[0].Args, "--threads")
|
||||||
|
}
|
||||||
|
if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "4" {
|
||||||
|
t.Errorf("--threads %q on 8 CPUs, expected 4", v)
|
||||||
|
}
|
||||||
|
if v, _ := threads(&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}, llamaSpec(nil)); v != "8" {
|
||||||
|
t.Errorf("--threads %q with 8 configured", v)
|
||||||
|
}
|
||||||
|
if _, ok := threads(&LlamaCpp{}, llamaSpec([]int{0})); ok {
|
||||||
|
t.Error("a GPU shard was given a thread count")
|
||||||
|
}
|
||||||
|
s := llamaSpec(nil)
|
||||||
|
s.Profile.Args = []string{"-t", "2"}
|
||||||
|
cmds, _ := (&LlamaCpp{}).Commands(s)
|
||||||
|
if slices.Contains(cmds[0].Args, "--threads") {
|
||||||
|
t.Error("the profile's own -t was overridden")
|
||||||
|
}
|
||||||
|
numCPU = func() int { return 1 }
|
||||||
|
if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "1" {
|
||||||
|
t.Errorf("--threads %q on one CPU", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) {
|
||||||
|
s := llamaSpec(nil)
|
||||||
|
s.Threads = 3
|
||||||
|
cmds, err := (&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}).Commands(s)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if v, _ := flagValue(cmds[0].Args, "--threads"); v != "3" {
|
||||||
|
t.Errorf("--threads %q; what placement reserved should win over the node's setting", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) {
|
||||||
|
s := llamaSpec(nil)
|
||||||
|
s.Profile.Adapters = []Adapter{{Name: "sql", Path: "/w/sql.gguf"}, {Name: "docs", Path: "/w/docs.gguf"}}
|
||||||
|
cmds, err := (&LlamaCpp{}).Commands(s)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
args := strings.Join(cmds[0].Args, " ")
|
||||||
|
if !strings.Contains(args, "--lora-scaled /w/sql.gguf:0 --lora-scaled /w/docs.gguf:0") {
|
||||||
|
t.Errorf("args %s: both adapters, in order, loaded at scale zero", args)
|
||||||
|
}
|
||||||
|
plain, _ := (&LlamaCpp{}).Commands(llamaSpec(nil))
|
||||||
|
if strings.Contains(strings.Join(plain[0].Args, " "), "--lora") {
|
||||||
|
t.Error("a model with no adapters was given lora flags")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -96,6 +96,11 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
args = append(args, p.Args...)
|
args = append(args, p.Args...)
|
||||||
|
// SGLang reads its key only from the command line, so on a node with
|
||||||
|
// other users it is visible to them in the process list.
|
||||||
|
if s.APIKey != "" && s.Leader() {
|
||||||
|
args = append(args, "--api-key", s.APIKey)
|
||||||
|
}
|
||||||
|
|
||||||
return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil
|
return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
package engines
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
// UnsafeEnv reports whether a profile may not set an environment variable,
|
||||||
|
// and why.
|
||||||
|
//
|
||||||
|
// A profile's env is the escape hatch for tuning no adapter can guess —
|
||||||
|
// CUDA_*, NCCL_*, VLLM_*, HF_* — and it reaches an engine the agent starts as
|
||||||
|
// root. Some variables are not tuning but a way to run other code in that
|
||||||
|
// process: the dynamic loader's LD_PRELOAD, Python's PYTHONSTARTUP, a shell's
|
||||||
|
// BASH_ENV. Set from a model definition, which an admin API or a config file
|
||||||
|
// supplies, they would turn "add a model" into "run this as root on every
|
||||||
|
// node". They are refused where a model is defined and again where an engine
|
||||||
|
// starts.
|
||||||
|
func UnsafeEnv(key string) (bool, string) {
|
||||||
|
k := strings.ToUpper(key)
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(k, "LD_"), strings.HasPrefix(k, "DYLD_"):
|
||||||
|
return true, "it changes what the dynamic loader loads"
|
||||||
|
case k == "PATH":
|
||||||
|
return true, "it changes which binaries run"
|
||||||
|
case strings.HasPrefix(k, "PYTHON") && k != "PYTHONUNBUFFERED" && k != "PYTHONHASHSEED":
|
||||||
|
return true, "it changes what Python imports or runs at start"
|
||||||
|
case k == "BASH_ENV" || k == "ENV" || k == "PERL5OPT" || k == "PERL5LIB" ||
|
||||||
|
k == "NODE_OPTIONS" || k == "RUBYOPT" || k == "GCONV_PATH" || k == "OPENSSL_CONF":
|
||||||
|
return true, "it makes an interpreter or library load code"
|
||||||
|
}
|
||||||
|
return false, ""
|
||||||
|
}
|
||||||
@@ -150,6 +150,9 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
|
|||||||
// Profile arguments last, so an operator can override anything above.
|
// Profile arguments last, so an operator can override anything above.
|
||||||
args = append(args, p.Args...)
|
args = append(args, p.Args...)
|
||||||
|
|
||||||
|
if s.APIKey != "" {
|
||||||
|
env["VLLM_API_KEY"] = s.APIKey
|
||||||
|
}
|
||||||
return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil
|
return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -166,7 +169,10 @@ func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) {
|
|||||||
|
|
||||||
args := []string{"start", "--block"}
|
args := []string{"start", "--block"}
|
||||||
if s.Leader() {
|
if s.Leader() {
|
||||||
args = append(args, "--head", "--port", strconv.Itoa(s.LeaderPort))
|
// Ray's head has no authentication: whoever reaches its port can run
|
||||||
|
// code on the node. Its dashboard, and the job API behind it, is the
|
||||||
|
// easiest way in, and nothing here uses it.
|
||||||
|
args = append(args, "--head", "--port", strconv.Itoa(s.LeaderPort), "--include-dashboard=false")
|
||||||
} else {
|
} else {
|
||||||
args = append(args, "--address", fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort))
|
args = append(args, "--address", fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,345 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// usableFabrics reports the fabrics this node can actually carry traffic on.
|
||||||
|
//
|
||||||
|
// Loud about what it drops. A fabric silently missing from a registration looks
|
||||||
|
// identical to one never configured, and the operator who typed the adapter
|
||||||
|
// name is the only person who can fix it — so the reason is logged here, on the
|
||||||
|
// machine where both halves are visible, rather than inferred later from a
|
||||||
|
// replica that will not converge.
|
||||||
|
func (a *Agent) usableFabrics(inv discovery.Inventory) []inventory.FabricAttachment {
|
||||||
|
out := a.checkFabrics(inv.HCAs, true)
|
||||||
|
a.mu.Lock()
|
||||||
|
a.fabricsSent = fabricsFingerprint(out)
|
||||||
|
a.mu.Unlock()
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkFabrics is the declared fabrics this node can carry traffic on, from
|
||||||
|
// the adapters as they are now. Loud logs every problem; the heartbeat checks
|
||||||
|
// quietly and speaks only when the answer changes.
|
||||||
|
func (a *Agent) checkFabrics(hcas []discovery.HCA, loud bool) []inventory.FabricAttachment {
|
||||||
|
declared := map[string][]string{}
|
||||||
|
for _, f := range a.cfg.Fabrics {
|
||||||
|
declared[f.Fabric] = f.HCAs
|
||||||
|
}
|
||||||
|
|
||||||
|
var out []inventory.FabricAttachment
|
||||||
|
for _, c := range discovery.CheckFabrics(declared, hcas) {
|
||||||
|
if loud {
|
||||||
|
for _, p := range c.Problems {
|
||||||
|
a.log.Error("a declared fabric adapter cannot carry traffic",
|
||||||
|
"fabric", c.Fabric, "problem", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !c.OK() {
|
||||||
|
if loud {
|
||||||
|
a.log.Error("fabric not advertised; no declared adapter on it can carry traffic",
|
||||||
|
"fabric", c.Fabric,
|
||||||
|
"consequence", "no multi-node replica will be placed here")
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, inventory.FabricAttachment{
|
||||||
|
Fabric: c.Fabric, HCAs: c.Usable, RateGbps: c.RateGbps,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// fabricsFingerprint is enough to notice a fabric, an adapter or a rate
|
||||||
|
// changing.
|
||||||
|
func fabricsFingerprint(fs []inventory.FabricAttachment) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, f := range fs {
|
||||||
|
fmt.Fprintf(&b, "%s:%s:%d;", f.Fabric, strings.Join(f.HCAs, ","), f.RateGbps)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// sampleIB re-reads the adapters for the heartbeat: every port's state and
|
||||||
|
// counters, and the fabrics again when they no longer match what the control
|
||||||
|
// plane was told.
|
||||||
|
//
|
||||||
|
// Every heartbeat, because it is a few small files per port, and because a
|
||||||
|
// port that went down used to be invisible until the agent restarted — the
|
||||||
|
// node went on being offered multi-node work that would hang on its first
|
||||||
|
// collective.
|
||||||
|
func (a *Agent) sampleIB(hb *llmgwv1.Heartbeat) {
|
||||||
|
a.mu.Lock()
|
||||||
|
had := len(a.inv.HCAs) > 0
|
||||||
|
a.mu.Unlock()
|
||||||
|
if !had && len(a.cfg.Fabrics) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
hcas, err := discovery.DiscoverIB(a.cfg.Discovery.IBRoot)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.attachSM(hcas)
|
||||||
|
for _, h := range hcas {
|
||||||
|
for _, p := range h.Ports {
|
||||||
|
ip := &llmgwv1.IBPort{
|
||||||
|
Hca: h.Name, Port: int32(p.Num), State: p.State, PhysState: p.PhysState,
|
||||||
|
RateGbps: int32(p.RateGbps), LinkLayer: string(p.LinkLayer), Counters: p.Counters,
|
||||||
|
NodeGuid: h.NodeGUID, PortGuid: p.PortGUID, Network: p.Network, GidPrefix: p.GIDPrefix,
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
if p.SM != nil {
|
||||||
|
ip.SmGuid, ip.SmLid, ip.SmPriority, ip.SmState = p.SM.GUID, int32(p.SM.LID), int32(p.SM.Priority), p.SM.State
|
||||||
|
} else {
|
||||||
|
ip.SmError = a.smErr[smKey(h.Name, p.Num)]
|
||||||
|
}
|
||||||
|
for _, r := range a.sms[smKey(h.Name, p.Num)] {
|
||||||
|
ip.SubnetManagers = append(ip.SubnetManagers, &llmgwv1.SubnetManager{
|
||||||
|
Guid: r.GUID, Lid: int32(r.LID), Priority: int32(r.Priority), State: r.State})
|
||||||
|
}
|
||||||
|
ip.SubnetManagersError = a.smsErr[smKey(h.Name, p.Num)]
|
||||||
|
a.mu.Unlock()
|
||||||
|
hb.IbPorts = append(hb.IbPorts, ip)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
a.inv.HCAs = hcas // a reconnect registers what is true now
|
||||||
|
a.mu.Unlock()
|
||||||
|
if len(a.cfg.Fabrics) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := a.checkFabrics(hcas, false)
|
||||||
|
fp := fabricsFingerprint(now)
|
||||||
|
a.mu.Lock()
|
||||||
|
changed := fp != a.fabricsSent
|
||||||
|
a.fabricsSent = fp
|
||||||
|
a.mu.Unlock()
|
||||||
|
if !changed {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Said once, with the reasons, the moment it happens.
|
||||||
|
a.checkFabrics(hcas, true)
|
||||||
|
a.log.Warn("the fabrics this node can carry traffic on changed", "now", fp)
|
||||||
|
hb.FabricsChanged = true
|
||||||
|
for _, f := range now {
|
||||||
|
hb.Fabrics = append(hb.Fabrics, &llmgwv1.FabricAttachment{
|
||||||
|
Fabric: f.Fabric, Hcas: f.HCAs, RateGbps: int32(f.RateGbps),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func smKey(hca string, port int) string { return fmt.Sprintf("%s/%d", hca, port) }
|
||||||
|
|
||||||
|
// scanFabric is discovery.ScanFabric, replaced in tests.
|
||||||
|
var scanFabric = discovery.ScanFabric
|
||||||
|
|
||||||
|
// querySMs is discovery.QuerySMs, replaced in tests.
|
||||||
|
var querySMs = func(hca string, port int) ([]discovery.SMRecord, error) {
|
||||||
|
return discovery.QuerySMs(context.Background(), hca, port)
|
||||||
|
}
|
||||||
|
|
||||||
|
func splitSMKey(k string) (string, int) {
|
||||||
|
i := strings.LastIndexByte(k, '/')
|
||||||
|
port, _ := strconv.Atoi(k[i+1:])
|
||||||
|
return k[:i], port
|
||||||
|
}
|
||||||
|
|
||||||
|
// querySM is discovery.QuerySM, replaced in tests.
|
||||||
|
var querySM = func(hca string, port int) (*discovery.SMInfo, error) {
|
||||||
|
return discovery.QuerySM(context.Background(), hca, port)
|
||||||
|
}
|
||||||
|
|
||||||
|
// attachSM puts on each active InfiniBand port the subnet manager it sees,
|
||||||
|
// asking again once a minute and otherwise reusing the last answer.
|
||||||
|
func (a *Agent) attachSM(hcas []discovery.HCA) {
|
||||||
|
a.mu.Lock()
|
||||||
|
stale := time.Since(a.smAsked) > time.Minute || a.sm == nil
|
||||||
|
a.mu.Unlock()
|
||||||
|
if stale {
|
||||||
|
sm, errs := map[string]*discovery.SMInfo{}, map[string]string{}
|
||||||
|
_ = discovery.AttachSM(hcas, func(hca string, port int) (*discovery.SMInfo, error) {
|
||||||
|
got, err := querySM(hca, port)
|
||||||
|
if err != nil {
|
||||||
|
errs[smKey(hca, port)] = err.Error()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sm[smKey(hca, port)] = got
|
||||||
|
return got, nil
|
||||||
|
})
|
||||||
|
// Every subnet manager on each fabric, through one port per master:
|
||||||
|
// the same list through every port of one fabric is one query too
|
||||||
|
// many for each.
|
||||||
|
sms, smsErr := map[string][]discovery.SMRecord{}, map[string]string{}
|
||||||
|
asked := map[string]bool{}
|
||||||
|
for k, v := range sm {
|
||||||
|
if asked[v.GUID] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
asked[v.GUID] = true
|
||||||
|
hca, port := splitSMKey(k)
|
||||||
|
if recs, err := querySMs(hca, port); err != nil {
|
||||||
|
smsErr[k] = err.Error()
|
||||||
|
} else {
|
||||||
|
sms[k] = recs
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
for k, v := range sm {
|
||||||
|
if old := a.sm[k]; old != nil && old.GUID != v.GUID {
|
||||||
|
a.log.Warn("the master subnet manager changed", "port", k, "from", old.GUID, "to", v.GUID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.sm, a.smErr, a.smAsked = sm, errs, time.Now()
|
||||||
|
a.sms, a.smsErr = sms, smsErr
|
||||||
|
a.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
defer a.mu.Unlock()
|
||||||
|
for i := range hcas {
|
||||||
|
for j := range hcas[i].Ports {
|
||||||
|
if sm := a.sm[smKey(hcas[i].Name, hcas[i].Ports[j].Num)]; sm != nil && hcas[i].Ports[j].Active() {
|
||||||
|
hcas[i].Ports[j].SM = sm
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensureDiags installs infiniband-diags on a node with adapters and without
|
||||||
|
// it. sminfo is what tells which physical fabric a port is on; without it,
|
||||||
|
// two separate fabrics given one name cannot be caught. Installed the way
|
||||||
|
// engine dependencies are, from the machine's own package sources.
|
||||||
|
func (a *Agent) ensureDiags(ctx context.Context) {
|
||||||
|
a.mu.Lock()
|
||||||
|
hasIB := len(a.inv.HCAs) > 0
|
||||||
|
a.mu.Unlock()
|
||||||
|
if !hasIB || a.prov == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err := exec.LookPath("sminfo"); err == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.log.Info("installing infiniband-diags, to tell which fabric each port is on")
|
||||||
|
pkgs := map[string][]string{"deb": {"infiniband-diags"}, "rpm": {"infiniband-diags"}}
|
||||||
|
if err := a.prov.InstallPackages(ctx, pkgs); err != nil {
|
||||||
|
a.log.Warn("could not install infiniband-diags; fabrics cannot be told apart by their subnet manager",
|
||||||
|
"err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// FeatureFabricTest is the agent answering FabricTestRequest.
|
||||||
|
const FeatureFabricTest = "fabric-test"
|
||||||
|
|
||||||
|
// runWriteBW is discovery.RunWriteBW, and haveWriteBW whether it is
|
||||||
|
// installed; both replaced in tests.
|
||||||
|
var (
|
||||||
|
runWriteBW = discovery.RunWriteBW
|
||||||
|
haveWriteBW = func() bool { _, err := exec.LookPath("ib_write_bw"); return err == nil }
|
||||||
|
)
|
||||||
|
|
||||||
|
// fabricTest runs this node's side of a bandwidth test with another.
|
||||||
|
func (a *Agent) fabricTest(ctx context.Context, req *llmgwv1.FabricTestRequest) *llmgwv1.FabricTestReply {
|
||||||
|
start := time.Now()
|
||||||
|
reply := &llmgwv1.FabricTestReply{RequestId: req.GetRequestId()}
|
||||||
|
defer func() { reply.TookMs = time.Since(start).Milliseconds() }()
|
||||||
|
|
||||||
|
w := discovery.WriteBW{HCA: req.GetHca(), Port: int(req.GetPort()), GIDIndex: -1,
|
||||||
|
TCPPort: int(req.GetTcpPort()), Seconds: int(req.GetSeconds())}
|
||||||
|
if req.GetRole() == "client" {
|
||||||
|
w.Peer = req.GetPeer()
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
found := false
|
||||||
|
for _, h := range a.inv.HCAs {
|
||||||
|
for _, p := range h.Ports {
|
||||||
|
if h.Name == w.HCA && p.Num == w.Port {
|
||||||
|
found = true
|
||||||
|
// RoCE needs its v2 GID named, as NCCL does; InfiniBand uses the LID.
|
||||||
|
if p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID >= 0 {
|
||||||
|
w.GIDIndex = p.RoCEv2GID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.mu.Unlock()
|
||||||
|
if !found {
|
||||||
|
reply.Error = fmt.Sprintf("this node has no port %s/%d", w.HCA, w.Port)
|
||||||
|
return reply
|
||||||
|
}
|
||||||
|
if !haveWriteBW() && a.prov != nil {
|
||||||
|
a.log.Info("installing perftest, for a bandwidth test with another node")
|
||||||
|
pkgs := map[string][]string{"deb": {"perftest"}, "rpm": {"perftest"}}
|
||||||
|
if err := a.prov.InstallPackages(ctx, pkgs); err != nil {
|
||||||
|
reply.Error = "installing perftest: " + err.Error()
|
||||||
|
return reply
|
||||||
|
}
|
||||||
|
}
|
||||||
|
gbps, out, err := runWriteBW(ctx, w)
|
||||||
|
reply.Gbps = gbps
|
||||||
|
reply.Output = tailLines(out, 12)
|
||||||
|
if err != nil {
|
||||||
|
reply.Error = err.Error()
|
||||||
|
}
|
||||||
|
return reply
|
||||||
|
}
|
||||||
|
|
||||||
|
func tailLines(s string, n int) string {
|
||||||
|
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
|
||||||
|
if len(lines) > n {
|
||||||
|
lines = lines[len(lines)-n:]
|
||||||
|
}
|
||||||
|
return strings.Join(lines, "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// discoverGPUFabric and discoverIMEX are replaced in tests.
|
||||||
|
var (
|
||||||
|
discoverGPUFabric = discovery.DiscoverGPUFabric
|
||||||
|
discoverIMEX = discovery.DiscoverIMEX
|
||||||
|
)
|
||||||
|
|
||||||
|
// sampleNVLink reads, once a minute, which NVLink domain each GPU is joined
|
||||||
|
// to and whether IMEX is up. nvidia-smi -q takes a second or two, too long
|
||||||
|
// for every heartbeat, and a fabric that was joined an hour ago rarely
|
||||||
|
// changes in five seconds — but when a fabric manager restarts it does, for
|
||||||
|
// every card at once.
|
||||||
|
func (a *Agent) sampleNVLink(hb *llmgwv1.Heartbeat) {
|
||||||
|
a.mu.Lock()
|
||||||
|
gpus := slices.Clone(a.inv.GPUs)
|
||||||
|
due := time.Since(a.nvlAsked) > time.Minute
|
||||||
|
if due {
|
||||||
|
a.nvlAsked = time.Now()
|
||||||
|
}
|
||||||
|
a.mu.Unlock()
|
||||||
|
if len(gpus) == 0 || !due {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
fab, err := discoverGPUFabric(ctx, a.cfg.Discovery.NvidiaSMI)
|
||||||
|
if err != nil || len(fab) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
discovery.ApplyFabric(gpus, fab)
|
||||||
|
for _, g := range gpus {
|
||||||
|
if g.Fabric == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
hb.GpuFabric = append(hb.GpuFabric, &llmgwv1.GPUFabricState{
|
||||||
|
Index: int32(g.Index), State: g.Fabric.State, Status: g.Fabric.Status,
|
||||||
|
ClusterUuid: g.Fabric.ClusterUUID, CliqueId: g.Fabric.CliqueID, Bandwidth: g.Fabric.Bandwidth,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
m := discoverIMEX(ctx)
|
||||||
|
hb.Imex = &llmgwv1.IMEX{State: m.State, Channels: int32(m.Channels)}
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
|
||||||
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
// logsReply answers a request for one of this node's logs.
|
||||||
|
func (a *Agent) logsReply(req *llmgwv1.LogsRequest) *llmgwv1.LogsReply {
|
||||||
|
n := int(req.GetLines())
|
||||||
|
if n <= 0 {
|
||||||
|
n = 200
|
||||||
|
}
|
||||||
|
n = min(n, agentLogLines)
|
||||||
|
shards := a.sup.IDs()
|
||||||
|
slices.Sort(shards)
|
||||||
|
out := &llmgwv1.LogsReply{
|
||||||
|
RequestId: req.GetRequestId(),
|
||||||
|
Sources: append([]string{"agent"}, shards...),
|
||||||
|
}
|
||||||
|
switch src := req.GetSource(); src {
|
||||||
|
case "", "agent":
|
||||||
|
out.Lines = a.logRing.tail(n)
|
||||||
|
default:
|
||||||
|
sh, err := a.sup.Get(src)
|
||||||
|
if err != nil {
|
||||||
|
out.Error = "no engine " + src + " runs on this node"
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
out.Lines = sh.Logs(n)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,197 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"maps"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
|
||||||
|
)
|
||||||
|
|
||||||
|
// engines is what this node can run: what the agent installed and saw run,
|
||||||
|
// and what the configuration names and answers. Nothing merely found on the
|
||||||
|
// machine — a llama-server on PATH is somebody's, not necessarily one that
|
||||||
|
// works, and placing on it is how a replica fails on a node nobody watches.
|
||||||
|
func (a *Agent) engines(ctx context.Context) map[string]string {
|
||||||
|
a.engineOnce.Do(func() {
|
||||||
|
vers := a.reg.ProbeConfigured(ctx)
|
||||||
|
a.mu.Lock()
|
||||||
|
a.configuredVers, a.lastEngineProbe = vers, time.Now()
|
||||||
|
a.mu.Unlock()
|
||||||
|
if len(vers) > 0 {
|
||||||
|
a.log.Info("engines named in the configuration",
|
||||||
|
"engines", engines.ProbedNames(vers))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
a.mu.Lock()
|
||||||
|
defer a.mu.Unlock()
|
||||||
|
vers := a.engineReportLocked()
|
||||||
|
// Whatever registration sends is what the control plane now knows, so a
|
||||||
|
// heartbeat repeats it only once it differs.
|
||||||
|
a.enginesSent = enginesFingerprint(vers)
|
||||||
|
return vers
|
||||||
|
}
|
||||||
|
|
||||||
|
// engineReportLocked merges the two sources. Installed wins: it is the one
|
||||||
|
// that was seen to run. Called with mu held.
|
||||||
|
func (a *Agent) engineReportLocked() map[string]string {
|
||||||
|
out := maps.Clone(a.configuredVers)
|
||||||
|
if out == nil {
|
||||||
|
out = map[string]string{}
|
||||||
|
}
|
||||||
|
maps.Copy(out, a.installedVers)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// provisionEvery is how often installed engines are checked again. A library
|
||||||
|
// removed by an unrelated upgrade is found here rather than by the next
|
||||||
|
// replica to start.
|
||||||
|
const provisionEvery = 10 * time.Minute
|
||||||
|
|
||||||
|
// setProvision takes the control plane's list of wanted engines.
|
||||||
|
func (a *Agent) setProvision(p *llmgwv1.Provision) {
|
||||||
|
wants := []provision.Want{}
|
||||||
|
for _, e := range p.GetEngines() {
|
||||||
|
wants = append(wants, provision.Want{Engine: e.GetEngine(), Version: e.GetVersion()})
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
a.provWants = wants
|
||||||
|
a.mu.Unlock()
|
||||||
|
select {
|
||||||
|
case a.provKick <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// provisionLoop installs and re-checks the wanted engines, one pass at a time.
|
||||||
|
func (a *Agent) provisionLoop(ctx context.Context) {
|
||||||
|
// After a failure the next pass comes sooner, and backs off: a lock held
|
||||||
|
// by a new machine's own updates clears in a minute or two, while ten
|
||||||
|
// minutes' wait left a node idle for no reason.
|
||||||
|
next := provisionEvery
|
||||||
|
retry := time.Minute
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-a.provKick:
|
||||||
|
case <-time.After(next):
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
wants := slices.Clone(a.provWants)
|
||||||
|
managed := a.provWants != nil
|
||||||
|
a.mu.Unlock()
|
||||||
|
if !managed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
statuses := a.prov.Converge(ctx, wants)
|
||||||
|
next = provisionEvery
|
||||||
|
for _, st := range statuses {
|
||||||
|
if st.State == provision.Failed {
|
||||||
|
next, retry = retry, min(retry*2, provisionEvery)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if next == provisionEvery {
|
||||||
|
retry = time.Minute
|
||||||
|
}
|
||||||
|
installed := map[string]string{}
|
||||||
|
keep := map[string]bool{}
|
||||||
|
for _, st := range statuses {
|
||||||
|
keep[st.Engine] = true
|
||||||
|
if st.State == provision.Ready {
|
||||||
|
a.reg.UseInstalled(st.Engine, st.Binary)
|
||||||
|
installed[st.Engine] = st.Version
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
if enginesFingerprint(installed) != enginesFingerprint(a.installedVers) {
|
||||||
|
a.log.Info("engines installed and running", "engines", engines.ProbedNames(installed))
|
||||||
|
}
|
||||||
|
a.installedVers = installed
|
||||||
|
// An engine no longer asked for is no longer reported on.
|
||||||
|
for e := range a.provStatus {
|
||||||
|
if !keep[e] {
|
||||||
|
delete(a.provStatus, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.mu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// provisionStatus records a change in one engine's installation.
|
||||||
|
func (a *Agent) provisionStatus(st provision.Status) {
|
||||||
|
a.mu.Lock()
|
||||||
|
prev, had := a.provStatus[st.Engine]
|
||||||
|
a.provStatus[st.Engine] = st
|
||||||
|
a.mu.Unlock()
|
||||||
|
if !had || prev.State != st.State || prev.Detail != st.Detail {
|
||||||
|
lvl := slog.LevelInfo
|
||||||
|
if st.State == provision.Failed || st.State == provision.Unsupported {
|
||||||
|
lvl = slog.LevelWarn
|
||||||
|
}
|
||||||
|
a.log.Log(context.Background(), lvl, "engine installation",
|
||||||
|
"engine", st.Engine, "version", st.Version, "variant", st.Variant,
|
||||||
|
"state", st.State, "detail", st.Detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func provisionFingerprint(m map[string]provision.Status) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, k := range slices.Sorted(maps.Keys(m)) {
|
||||||
|
st := m[k]
|
||||||
|
fmt.Fprintf(&b, "%s|%s|%s|%s|%s\n", st.Engine, st.Version, st.Variant, st.State, st.Detail)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// engineReprobe is how often the installed engines are looked for again.
|
||||||
|
//
|
||||||
|
// Minutes, not seconds: an engine is installed by somebody at a keyboard, so a
|
||||||
|
// minute's delay in noticing is invisible, while SGLang's probe starts a
|
||||||
|
// Python interpreter that imports it.
|
||||||
|
const engineReprobe = time.Minute
|
||||||
|
|
||||||
|
// reprobeEngines looks for installed engines again when it is time, off the
|
||||||
|
// heartbeat goroutine so a slow probe never delays a heartbeat.
|
||||||
|
func (a *Agent) reprobeEngines() {
|
||||||
|
a.mu.Lock()
|
||||||
|
due := !a.probingEngines && !a.lastEngineProbe.IsZero() &&
|
||||||
|
time.Since(a.lastEngineProbe) >= engineReprobe
|
||||||
|
if due {
|
||||||
|
a.probingEngines = true
|
||||||
|
}
|
||||||
|
a.mu.Unlock()
|
||||||
|
if !due {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
vers := a.reg.ProbeConfigured(ctx)
|
||||||
|
|
||||||
|
a.mu.Lock()
|
||||||
|
changed := enginesFingerprint(vers) != enginesFingerprint(a.configuredVers)
|
||||||
|
a.configuredVers, a.lastEngineProbe, a.probingEngines = vers, time.Now(), false
|
||||||
|
a.mu.Unlock()
|
||||||
|
if changed {
|
||||||
|
a.log.Info("engines named in the configuration changed",
|
||||||
|
"engines", engines.ProbedNames(vers))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
func enginesFingerprint(vers map[string]string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, k := range slices.Sorted(maps.Keys(vers)) {
|
||||||
|
fmt.Fprintf(&b, "%s=%s\n", k, vers[k])
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -77,6 +78,10 @@ type Installer struct {
|
|||||||
PyPIBase string
|
PyPIBase string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// validVersion is what a version may look like: "b10991", "0.29.0",
|
||||||
|
// "0.11.0+cu128". No separators, and not starting with a dot.
|
||||||
|
var validVersion = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$`)
|
||||||
|
|
||||||
// DefaultRoot is where engines are installed when nothing says otherwise.
|
// DefaultRoot is where engines are installed when nothing says otherwise.
|
||||||
const DefaultRoot = "/opt/llmgw/engines"
|
const DefaultRoot = "/opt/llmgw/engines"
|
||||||
|
|
||||||
@@ -114,6 +119,15 @@ func (in *Installer) Converge(ctx context.Context, wants []Want) []Status {
|
|||||||
|
|
||||||
func (in *Installer) one(ctx context.Context, w Want) Status {
|
func (in *Installer) one(ctx context.Context, w Want) Status {
|
||||||
st := Status{Engine: w.Engine, Version: w.Version}
|
st := Status{Engine: w.Engine, Version: w.Version}
|
||||||
|
// Checked here, on the node, whatever the control plane checked: the
|
||||||
|
// version names a directory under the install root and is passed to a
|
||||||
|
// package manager, and "../x" would put an install — and the RemoveAll
|
||||||
|
// that replaces one — outside it, as root.
|
||||||
|
if !validVersion.MatchString(w.Version) {
|
||||||
|
st.State = Unsupported
|
||||||
|
st.Detail = fmt.Sprintf("%q is not a version this agent will install", w.Version)
|
||||||
|
return st
|
||||||
|
}
|
||||||
recipe, ok := Recipes[w.Engine]
|
recipe, ok := Recipes[w.Engine]
|
||||||
if !ok {
|
if !ok {
|
||||||
st.State = Unsupported
|
st.State = Unsupported
|
||||||
@@ -509,6 +523,11 @@ func (in *Installer) download(ctx context.Context, repo, tag, asset, dst, want s
|
|||||||
// untarGz unpacks an archive under dir, refusing anything that would land
|
// untarGz unpacks an archive under dir, refusing anything that would land
|
||||||
// outside it. A release archive is somebody else's file, and a path of
|
// outside it. A release archive is somebody else's file, and a path of
|
||||||
// "../../etc/cron.d/x" in it would otherwise be written as root.
|
// "../../etc/cron.d/x" in it would otherwise be written as root.
|
||||||
|
//
|
||||||
|
// Every write goes through an os.Root, which refuses to leave dir however the
|
||||||
|
// path gets there. Checking names alone was not enough: "a -> ." followed by
|
||||||
|
// "a/a/b -> ../.." reads as inside and resolves two levels above it, and a
|
||||||
|
// file written through that lands wherever the links point.
|
||||||
func untarGz(archive, dir string) error {
|
func untarGz(archive, dir string) error {
|
||||||
f, err := os.Open(archive) //nolint:gosec // inside the install root
|
f, err := os.Open(archive) //nolint:gosec // inside the install root
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -519,6 +538,11 @@ func untarGz(archive, dir string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
root, err := os.OpenRoot(dir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer root.Close()
|
||||||
tr := tar.NewReader(gz)
|
tr := tar.NewReader(gz)
|
||||||
for {
|
for {
|
||||||
h, err := tr.Next()
|
h, err := tr.Next()
|
||||||
@@ -528,22 +552,22 @@ func untarGz(archive, dir string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
target, err := within(dir, h.Name)
|
name := filepath.Clean(h.Name)
|
||||||
if err != nil {
|
if _, err := within(dir, name); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
switch h.Typeflag {
|
switch h.Typeflag {
|
||||||
case tar.TypeDir:
|
case tar.TypeDir:
|
||||||
if err := os.MkdirAll(target, 0o755); err != nil {
|
if err := root.MkdirAll(name, 0o755); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
case tar.TypeReg:
|
case tar.TypeReg:
|
||||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
if err := root.MkdirAll(filepath.Dir(name), 0o755); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755) //nolint:gosec // checked above
|
out, err := root.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return fmt.Errorf("%s: %w", h.Name, err)
|
||||||
}
|
}
|
||||||
if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // a release archive, size bounded by its checksum
|
if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // a release archive, size bounded by its checksum
|
||||||
out.Close()
|
out.Close()
|
||||||
@@ -553,19 +577,19 @@ func untarGz(archive, dir string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
case tar.TypeSymlink:
|
case tar.TypeSymlink:
|
||||||
// Libraries are versioned through symlinks, and those are
|
// Libraries are versioned through symlinks to a file beside them —
|
||||||
// relative. One pointing anywhere else is refused.
|
// libllama.so -> libllama.so.0 — and that is all a release needs.
|
||||||
if filepath.IsAbs(h.Linkname) {
|
// A target with a separator in it could point anywhere once other
|
||||||
return fmt.Errorf("%s links outside the archive", h.Name)
|
// links are followed, and a binary found through one would be run
|
||||||
|
// as root, so it is refused rather than reasoned about.
|
||||||
|
if h.Linkname == "" || strings.ContainsAny(h.Linkname, `/\`) || h.Linkname == ".." || h.Linkname == "." {
|
||||||
|
return fmt.Errorf("%s links to %q; only a file beside it is allowed", h.Name, h.Linkname)
|
||||||
}
|
}
|
||||||
if _, err := within(dir, filepath.Join(filepath.Dir(h.Name), h.Linkname)); err != nil {
|
if err := root.MkdirAll(filepath.Dir(name), 0o755); err != nil {
|
||||||
return fmt.Errorf("%s links outside the archive", h.Name)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
_ = os.Remove(target)
|
_ = root.Remove(name)
|
||||||
if err := os.Symlink(h.Linkname, target); err != nil {
|
if err := root.Symlink(h.Linkname, name); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
|
|||||||
@@ -330,3 +330,61 @@ func TestAPackageManagerBusyWithItsOwnUpdatesIsWaitedFor(t *testing.T) {
|
|||||||
t.Error("the wait was not reported")
|
t.Error("the wait was not reported")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAVersionThatIsAPathIsRefused(t *testing.T) {
|
||||||
|
// The version names a directory under the install root; "../" would put
|
||||||
|
// an install, and the RemoveAll that replaces one, outside it, as root.
|
||||||
|
in := &Installer{Root: t.TempDir(), Host: ubuntu}
|
||||||
|
for _, v := range []string{"../../etc", "..", ".hidden", "b1/../../x", "", "b1 && rm -rf /"} {
|
||||||
|
got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: v}})
|
||||||
|
if got[0].State != Unsupported {
|
||||||
|
t.Errorf("version %q: %+v", v, got[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChainedLinksCannotCarryAFileOutOfTheInstall(t *testing.T) {
|
||||||
|
// Each link reads as inside the directory; together they point above it.
|
||||||
|
var buf bytes.Buffer
|
||||||
|
gz := gzip.NewWriter(&buf)
|
||||||
|
tw := tar.NewWriter(gz)
|
||||||
|
_ = tw.WriteHeader(&tar.Header{Name: "a", Linkname: ".", Typeflag: tar.TypeSymlink})
|
||||||
|
_ = tw.WriteHeader(&tar.Header{Name: "a/a/b", Linkname: "../..", Typeflag: tar.TypeSymlink})
|
||||||
|
_ = tw.WriteHeader(&tar.Header{Name: "a/a/b/escaped", Mode: 0o644, Size: 1, Typeflag: tar.TypeReg})
|
||||||
|
_, _ = tw.Write([]byte("x"))
|
||||||
|
_ = tw.Close()
|
||||||
|
_ = gz.Close()
|
||||||
|
|
||||||
|
parent := t.TempDir()
|
||||||
|
dir := filepath.Join(parent, "install")
|
||||||
|
_ = os.MkdirAll(dir, 0o755)
|
||||||
|
archive := filepath.Join(parent, "evil.tar.gz")
|
||||||
|
_ = os.WriteFile(archive, buf.Bytes(), 0o644)
|
||||||
|
|
||||||
|
if err := untarGz(archive, dir); err == nil {
|
||||||
|
t.Error("an archive of chained links was unpacked")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(parent, "escaped")); err == nil {
|
||||||
|
t.Fatal("a file was written outside the install directory")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestALibrarysVersionLinkIsStillUnpacked(t *testing.T) {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
gz := gzip.NewWriter(&buf)
|
||||||
|
tw := tar.NewWriter(gz)
|
||||||
|
_ = tw.WriteHeader(&tar.Header{Name: "lib/libllama.so.0", Mode: 0o755, Size: 1, Typeflag: tar.TypeReg})
|
||||||
|
_, _ = tw.Write([]byte("x"))
|
||||||
|
_ = tw.WriteHeader(&tar.Header{Name: "lib/libllama.so", Linkname: "libllama.so.0", Typeflag: tar.TypeSymlink})
|
||||||
|
_ = tw.Close()
|
||||||
|
_ = gz.Close()
|
||||||
|
dir := t.TempDir()
|
||||||
|
archive := filepath.Join(t.TempDir(), "ok.tar.gz")
|
||||||
|
_ = os.WriteFile(archive, buf.Bytes(), 0o644)
|
||||||
|
if err := untarGz(archive, dir); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if target, err := os.Readlink(filepath.Join(dir, "lib", "libllama.so")); err != nil || target != "libllama.so.0" {
|
||||||
|
t.Errorf("link %q %v", target, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -53,8 +53,8 @@ func NewHTTPHealth(timeout time.Duration, opts ...HTTPHealthOption) *HTTPHealth
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Ready reports whether the engine at baseURL is serving model.
|
// Ready reports whether the engine at baseURL is serving model.
|
||||||
func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
|
func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model, apiKey string) error {
|
||||||
if err := h.get(ctx, baseURL+h.healthPath, nil); err != nil {
|
if err := h.get(ctx, baseURL+h.healthPath, apiKey, nil); err != nil {
|
||||||
return fmt.Errorf("health: %w", err)
|
return fmt.Errorf("health: %w", err)
|
||||||
}
|
}
|
||||||
if !h.requireName || model == "" {
|
if !h.requireName || model == "" {
|
||||||
@@ -66,7 +66,7 @@ func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
|
|||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
} `json:"data"`
|
} `json:"data"`
|
||||||
}
|
}
|
||||||
if err := h.get(ctx, baseURL+h.modelsPath, &list); err != nil {
|
if err := h.get(ctx, baseURL+h.modelsPath, apiKey, &list); err != nil {
|
||||||
return fmt.Errorf("models: %w", err)
|
return fmt.Errorf("models: %w", err)
|
||||||
}
|
}
|
||||||
for _, d := range list.Data {
|
for _, d := range list.Data {
|
||||||
@@ -77,11 +77,14 @@ func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
|
|||||||
return fmt.Errorf("engine is up but does not serve %q yet", model)
|
return fmt.Errorf("engine is up but does not serve %q yet", model)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *HTTPHealth) get(ctx context.Context, url string, out any) error {
|
func (h *HTTPHealth) get(ctx context.Context, url, apiKey string, out any) error {
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if apiKey != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||||
|
}
|
||||||
resp, err := h.client.Do(req)
|
resp, err := h.client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ func engineStub(t *testing.T, healthy bool, models ...string) *httptest.Server {
|
|||||||
|
|
||||||
func TestReadyWhenServingModel(t *testing.T) {
|
func TestReadyWhenServingModel(t *testing.T) {
|
||||||
srv := engineStub(t, true, "llama-3.3-70b")
|
srv := engineStub(t, true, "llama-3.3-70b")
|
||||||
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b"); err != nil {
|
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", ""); err != nil {
|
||||||
t.Errorf("a serving engine was reported unready: %v", err)
|
t.Errorf("a serving engine was reported unready: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -48,7 +48,7 @@ func TestNotReadyWhileLoading(t *testing.T) {
|
|||||||
// The case that matters: the process is up but the weights are not loaded.
|
// The case that matters: the process is up but the weights are not loaded.
|
||||||
// Routing on /health alone would send traffic into a loading model.
|
// Routing on /health alone would send traffic into a loading model.
|
||||||
srv := engineStub(t, false, "llama-3.3-70b")
|
srv := engineStub(t, false, "llama-3.3-70b")
|
||||||
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b")
|
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", "")
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("an unhealthy engine was reported ready")
|
t.Fatal("an unhealthy engine was reported ready")
|
||||||
}
|
}
|
||||||
@@ -61,7 +61,7 @@ func TestNotReadyWhenModelMissing(t *testing.T) {
|
|||||||
// /health passes but the model is not listed: the engine is up and serving
|
// /health passes but the model is not listed: the engine is up and serving
|
||||||
// something else, or still registering.
|
// something else, or still registering.
|
||||||
srv := engineStub(t, true, "some-other-model")
|
srv := engineStub(t, true, "some-other-model")
|
||||||
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b")
|
err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", "")
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("a model that is not served was reported ready")
|
t.Fatal("a model that is not served was reported ready")
|
||||||
}
|
}
|
||||||
@@ -73,14 +73,33 @@ func TestNotReadyWhenModelMissing(t *testing.T) {
|
|||||||
func TestModelCheckCanBeSkipped(t *testing.T) {
|
func TestModelCheckCanBeSkipped(t *testing.T) {
|
||||||
srv := engineStub(t, true)
|
srv := engineStub(t, true)
|
||||||
h := NewHTTPHealth(time.Second, WithoutModelCheck())
|
h := NewHTTPHealth(time.Second, WithoutModelCheck())
|
||||||
if err := h.Ready(t.Context(), srv.URL, "anything"); err != nil {
|
if err := h.Ready(t.Context(), srv.URL, "anything", ""); err != nil {
|
||||||
t.Errorf("the model check was not skipped: %v", err)
|
t.Errorf("the model check was not skipped: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUnreachableEngine(t *testing.T) {
|
func TestUnreachableEngine(t *testing.T) {
|
||||||
h := NewHTTPHealth(200 * time.Millisecond)
|
h := NewHTTPHealth(200 * time.Millisecond)
|
||||||
if err := h.Ready(t.Context(), "http://127.0.0.1:1", "m"); err == nil {
|
if err := h.Ready(t.Context(), "http://127.0.0.1:1", "m", ""); err == nil {
|
||||||
t.Error("an unreachable engine was reported ready")
|
t.Error("an unreachable engine was reported ready")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTheHealthCheckSendsTheEnginesKey(t *testing.T) {
|
||||||
|
// An engine started with a key refuses /v1/models without it, and a
|
||||||
|
// shard that is serving would read as never ready.
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Header.Get("Authorization") != "Bearer k1" {
|
||||||
|
w.WriteHeader(http.StatusUnauthorized)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"data":[{"id":"m"}]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "m", "k1"); err != nil {
|
||||||
|
t.Errorf("with the key: %v", err)
|
||||||
|
}
|
||||||
|
if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "m", ""); err == nil {
|
||||||
|
t.Error("without the key the engine refused, and that read as ready")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -79,7 +79,7 @@ func (c *Config) normalize() {
|
|||||||
// supervisor can be tested without an HTTP engine.
|
// supervisor can be tested without an HTTP engine.
|
||||||
type HealthChecker interface {
|
type HealthChecker interface {
|
||||||
// Ready reports whether the shard at baseURL serves model.
|
// Ready reports whether the shard at baseURL serves model.
|
||||||
Ready(ctx context.Context, baseURL, model string) error
|
Ready(ctx context.Context, baseURL, model, apiKey string) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Shard is one supervised engine process.
|
// Shard is one supervised engine process.
|
||||||
@@ -411,7 +411,7 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
|
|||||||
last := strings.Join(sh.Logs(10), "\n")
|
last := strings.Join(sh.Logs(10), "\n")
|
||||||
return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last)
|
return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last)
|
||||||
case <-t.C:
|
case <-t.C:
|
||||||
if err := s.health.Ready(ctx, baseURL, model); err != nil {
|
if err := s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
sh.setState(Ready, nil)
|
sh.setState(Ready, nil)
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import (
|
|||||||
// scriptedHealth reports ready once the flag is set.
|
// scriptedHealth reports ready once the flag is set.
|
||||||
type scriptedHealth struct{ ready atomic.Bool }
|
type scriptedHealth struct{ ready atomic.Bool }
|
||||||
|
|
||||||
func (h *scriptedHealth) Ready(context.Context, string, string) error {
|
func (h *scriptedHealth) Ready(context.Context, string, string, string) error {
|
||||||
if h.ready.Load() {
|
if h.ready.Load() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -243,7 +243,9 @@ func TestWaitReadyTimeoutIncludesLogs(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
time.Sleep(100 * time.Millisecond) // let the line be captured
|
eventually(t, 3*time.Second, "the line was never captured", func() bool {
|
||||||
|
return strings.Contains(strings.Join(sh.Logs(10), "\n"), "CUDA out of memory")
|
||||||
|
})
|
||||||
|
|
||||||
err = s.WaitReady(t.Context(), sh, "http://x", "m", 200*time.Millisecond)
|
err = s.WaitReady(t.Context(), sh, "http://x", "m", 200*time.Millisecond)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -295,7 +297,10 @@ func TestLogsAreCaptured(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
time.Sleep(400 * time.Millisecond)
|
eventually(t, 3*time.Second, "not every line arrived", func() bool {
|
||||||
|
l := strings.Join(sh.Logs(10), "\n")
|
||||||
|
return strings.Contains(l, "line1") && strings.Contains(l, "line2") && strings.Contains(l, "line3")
|
||||||
|
})
|
||||||
|
|
||||||
logs := strings.Join(sh.Logs(10), "\n")
|
logs := strings.Join(sh.Logs(10), "\n")
|
||||||
for _, want := range []string{"line1", "line2", "line3"} {
|
for _, want := range []string{"line1", "line2", "line3"} {
|
||||||
@@ -372,11 +377,11 @@ func TestStateFileRemovedOnExit(t *testing.T) {
|
|||||||
|
|
||||||
sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 0"))
|
sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 0"))
|
||||||
waitState(t, sh, Failed, 3*time.Second)
|
waitState(t, sh, Failed, 3*time.Second)
|
||||||
time.Sleep(50 * time.Millisecond)
|
eventually(t, 3*time.Second,
|
||||||
|
"the state file outlived the process; a later scan would find a dead orphan", func() bool {
|
||||||
if _, err := os.Stat(s.statePath("a")); !os.IsNotExist(err) {
|
_, err := os.Stat(s.statePath("a"))
|
||||||
t.Error("the state file outlived the process; a later scan would find a dead orphan")
|
return os.IsNotExist(err)
|
||||||
}
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- state machine and ring buffer ---
|
// --- state machine and ring buffer ---
|
||||||
@@ -743,7 +748,9 @@ func TestWaitAliveFailsOnEarlyExit(t *testing.T) {
|
|||||||
if err := s.WaitAlive(t.Context(), sh); err == nil {
|
if err := s.WaitAlive(t.Context(), sh); err == nil {
|
||||||
t.Fatal("a worker that died immediately was reported ready")
|
t.Fatal("a worker that died immediately was reported ready")
|
||||||
}
|
}
|
||||||
if d := time.Since(start); d > 1500*time.Millisecond {
|
// Below AliveGrace (5s), which is what waiting it out would take; the
|
||||||
|
// headroom is for a loaded runner, where 1.5s was sometimes not enough.
|
||||||
|
if d := time.Since(start); d > 4*time.Second {
|
||||||
t.Errorf("waited %v for a process that had already exited", d)
|
t.Errorf("waited %v for a process that had already exited", d)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -827,3 +834,17 @@ func TestAWorkersOnlyProcessWaitsForTheLeader(t *testing.T) {
|
|||||||
t.Errorf("the worker started after %v, before the leader was listening", d)
|
t.Errorf("the worker started after %v, before the leader was listening", d)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// eventually waits for cond, failing with msg if it does not hold within
|
||||||
|
// limit. Sleeping a fixed time and then checking measured the machine the
|
||||||
|
// test ran on, and a loaded CI runner under the race detector lost that race.
|
||||||
|
func eventually(t *testing.T, limit time.Duration, msg string, cond func() bool) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(limit)
|
||||||
|
for !cond() {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal(msg)
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||||
|
|
||||||
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
// upgradeExit is the exit status after an upgrade: non-zero, so systemd's
|
||||||
|
// Restart=on-failure starts the new binary, and distinct, so the journal says
|
||||||
|
// why the agent went.
|
||||||
|
const upgradeExit = 75
|
||||||
|
|
||||||
|
// exitForUpgrade ends the process; a variable so a test can see it called
|
||||||
|
// rather than have its own process end.
|
||||||
|
var exitForUpgrade = func() { os.Exit(upgradeExit) }
|
||||||
|
|
||||||
|
// upgradeTarget is the binary an upgrade replaces; a variable so a test
|
||||||
|
// replaces a file of its own rather than the test binary running it.
|
||||||
|
var upgradeTarget = os.Executable
|
||||||
|
|
||||||
|
// upgrade replaces this agent's binary with a release and restarts into it.
|
||||||
|
//
|
||||||
|
// Engines are not touched: an agent restart leaves them running and adopts
|
||||||
|
// them again, which is what makes upgrading a node something that can be done
|
||||||
|
// while it serves.
|
||||||
|
func (a *Agent) upgrade(ctx context.Context, m *llmgwv1.UpgradeAgent, out chan<- *llmgwv1.AgentMessage) {
|
||||||
|
reply := &llmgwv1.UpgradeReply{RequestId: m.GetRequestId(), Version: m.GetVersion()}
|
||||||
|
send := func() {
|
||||||
|
select {
|
||||||
|
case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: reply}}:
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.log.Info("upgrading", "to", m.GetVersion(), "from", a.cfg.Version)
|
||||||
|
self, err := upgradeTarget()
|
||||||
|
if err == nil {
|
||||||
|
dctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||||
|
_, err = release.Install(dctx, nil, m.GetReleases(), m.GetVersion(), runtime.GOOS, runtime.GOARCH, self)
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
reply.Error = err.Error()
|
||||||
|
a.log.Error("upgrade failed; still running the old version", "to", m.GetVersion(), "err", err)
|
||||||
|
send()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
send()
|
||||||
|
a.log.Info("upgraded; restarting into the new version", "to", m.GetVersion())
|
||||||
|
// A moment for the reply to leave before the stream goes.
|
||||||
|
time.Sleep(500 * time.Millisecond)
|
||||||
|
exitForUpgrade()
|
||||||
|
}
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAnAgentUpgradesItselfAndSaysSoBeforeRestarting(t *testing.T) {
|
||||||
|
body := []byte("the new agent")
|
||||||
|
sum := sha256.Sum256(body)
|
||||||
|
asset := release.AssetName("v9.9.9", runtime.GOOS, runtime.GOARCH)
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch r.URL.Path {
|
||||||
|
case "/v9.9.9/checksums.txt":
|
||||||
|
_, _ = w.Write([]byte(hex.EncodeToString(sum[:]) + " " + asset + "\n"))
|
||||||
|
case "/v9.9.9/" + asset:
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
default:
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
bin := filepath.Join(t.TempDir(), "llmgw")
|
||||||
|
_ = os.WriteFile(bin, []byte("the old agent"), 0o755)
|
||||||
|
exited := make(chan struct{}, 1)
|
||||||
|
oldTarget, oldExit := upgradeTarget, exitForUpgrade
|
||||||
|
upgradeTarget = func() (string, error) { return bin, nil }
|
||||||
|
exitForUpgrade = func() { exited <- struct{}{} }
|
||||||
|
t.Cleanup(func() { upgradeTarget, exitForUpgrade = oldTarget, oldExit })
|
||||||
|
|
||||||
|
cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(), EnginesDir: t.TempDir()}
|
||||||
|
cfg.Superv.WorkDir = t.TempDir()
|
||||||
|
a, err := New(cfg, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out := make(chan *llmgwv1.AgentMessage, 1)
|
||||||
|
go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u1", Version: "v9.9.9", Releases: srv.URL}, out)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case m := <-out:
|
||||||
|
if r := m.GetUpgrade(); r.GetRequestId() != "u1" || r.GetError() != "" {
|
||||||
|
t.Fatalf("reply %+v", r)
|
||||||
|
}
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("no reply")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-exited:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the agent did not restart into the new binary")
|
||||||
|
}
|
||||||
|
if b, _ := os.ReadFile(bin); string(b) != "the new agent" {
|
||||||
|
t.Errorf("binary holds %q", b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A version with no release answers with why, and does not restart.
|
||||||
|
go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u2", Version: "v9.9.8", Releases: srv.URL}, out)
|
||||||
|
if r := (<-out).GetUpgrade(); r.GetError() == "" {
|
||||||
|
t.Error("an upgrade to a release that does not exist reported success")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-exited:
|
||||||
|
t.Error("a failed upgrade restarted the agent")
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -225,6 +225,7 @@ func runStep(ctx context.Context, client *http.Client, cfg Config, concurrency i
|
|||||||
)
|
)
|
||||||
|
|
||||||
sampler := newSampler(cfg.MetricsURL, cfg.Engine)
|
sampler := newSampler(cfg.MetricsURL, cfg.Engine)
|
||||||
|
sampler.apiKey = cfg.APIKey
|
||||||
sampler.start(ctx)
|
sampler.start(ctx)
|
||||||
|
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import (
|
|||||||
// Optional, because a run pointed at the gateway has no single engine to read.
|
// Optional, because a run pointed at the gateway has no single engine to read.
|
||||||
type sampler struct {
|
type sampler struct {
|
||||||
url string
|
url string
|
||||||
|
apiKey string
|
||||||
engine string
|
engine string
|
||||||
client *http.Client
|
client *http.Client
|
||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
@@ -65,6 +66,9 @@ func (s *sampler) sample(ctx context.Context) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if s.apiKey != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+s.apiKey)
|
||||||
|
}
|
||||||
resp, err := s.client.Do(req)
|
resp, err := s.client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -0,0 +1,312 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"compress/gzip"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/fs"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DefaultBackupDir is where backups are written and where the control plane
|
||||||
|
// looks for the newest one.
|
||||||
|
//
|
||||||
|
// Its own directory rather than /var/backups/llmgw, which a PostgreSQL move
|
||||||
|
// had already made for its own dumps, owned by postgres and closed to
|
||||||
|
// everyone else — so the control plane could not see a backup landing in it.
|
||||||
|
const DefaultBackupDir = "/var/backups/llmgw-server"
|
||||||
|
|
||||||
|
func newBackupCmd() *cobra.Command {
|
||||||
|
var (
|
||||||
|
configPath string
|
||||||
|
etcDir string
|
||||||
|
dir string
|
||||||
|
keep int
|
||||||
|
)
|
||||||
|
c := &cobra.Command{
|
||||||
|
Use: "backup",
|
||||||
|
Short: "Write the store and /etc/llmgw to one archive, and keep the last few",
|
||||||
|
Long: "Everything a control plane cannot be rebuilt without: the store — keys,\n" +
|
||||||
|
"people, quotas, usage, the models added at runtime, the secret engine\n" +
|
||||||
|
"keys derive from — and /etc/llmgw, which holds the configuration, the\n" +
|
||||||
|
"store's address and the certificate authority's key.\n\n" +
|
||||||
|
"The archive holds that key, so it is written readable by root only. It\n" +
|
||||||
|
"is on the same disk as what it backs up: copy it elsewhere, or it does\n" +
|
||||||
|
"not survive the loss of this machine.\n\n" +
|
||||||
|
"Run daily by llmgw-backup.timer (`llmgw server backup`). The control plane publishes the time of\n" +
|
||||||
|
"the newest archive as llmgw_backup_last_success_timestamp_seconds, and\n" +
|
||||||
|
"BackupTooOld fires when it is more than a day old.",
|
||||||
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
cfg, err := config.Load(configPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
path, err := writeBackup(cmd.Context(), cfg.Auth.Store, etcDir, dir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if why := unlistable(dir); why != "" {
|
||||||
|
fmt.Fprintf(cmd.ErrOrStderr(), "warning: %s; the control plane will report no backup "+
|
||||||
|
"and BackupTooOld will fire\n", why)
|
||||||
|
}
|
||||||
|
removed, err := pruneBackups(dir, keep)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "wrote %s", path)
|
||||||
|
if removed > 0 {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "; removed %d older", removed)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(cmd.OutOrStdout())
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c.Flags().StringVar(&configPath, "config", "/etc/llmgw/server.yaml", "the control plane's configuration")
|
||||||
|
c.Flags().StringVar(&etcDir, "etc", "/etc/llmgw", "configuration directory to include")
|
||||||
|
c.Flags().StringVar(&dir, "dir", DefaultBackupDir, "where archives are written")
|
||||||
|
c.Flags().IntVar(&keep, "keep", 14, "how many archives to keep")
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
const backupPrefix = "llmgw-backup-"
|
||||||
|
|
||||||
|
// writeBackup writes one archive and returns its path. Written beside its
|
||||||
|
// final name and renamed, so a backup interrupted halfway is never the newest
|
||||||
|
// one the control plane reports.
|
||||||
|
func writeBackup(ctx context.Context, storeAddr, etcDir, dir string) (string, error) {
|
||||||
|
// Listable by anyone, so the control plane — not root — can see when the
|
||||||
|
// newest archive was written. The archives themselves are root's alone.
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
host, _ := os.Hostname()
|
||||||
|
name := backupPrefix + host + "-" + time.Now().UTC().Format("20060102T150405Z") + ".tar.gz"
|
||||||
|
final := filepath.Join(dir, name)
|
||||||
|
part := final + ".part"
|
||||||
|
|
||||||
|
f, err := os.OpenFile(part, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
ok := false
|
||||||
|
defer func() {
|
||||||
|
if !ok {
|
||||||
|
_ = os.Remove(part)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
gz := gzip.NewWriter(f)
|
||||||
|
tw := tar.NewWriter(gz)
|
||||||
|
|
||||||
|
manifest := fmt.Sprintf("llmgw %s\nhost %s\ntime %s\n", version.Version, host,
|
||||||
|
time.Now().UTC().Format(time.RFC3339))
|
||||||
|
if err := addBytes(tw, "MANIFEST", []byte(manifest)); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if storeAddr != "" {
|
||||||
|
if err := addStore(ctx, tw, storeAddr, dir); err != nil {
|
||||||
|
return "", fmt.Errorf("the store: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := addTree(tw, etcDir, "etc/llmgw"); err != nil {
|
||||||
|
return "", fmt.Errorf("%s: %w", etcDir, err)
|
||||||
|
}
|
||||||
|
if err := errors.Join(tw.Close(), gz.Close(), f.Sync(), f.Close()); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := os.Rename(part, final); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
ok = true
|
||||||
|
return final, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// addStore dumps the store into the archive.
|
||||||
|
func addStore(ctx context.Context, tw *tar.Writer, addr, scratch string) error {
|
||||||
|
addr, err := store.ResolveAddress(addr)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Inside a directory only its owner can enter. The archive directory is
|
||||||
|
// listable on purpose, and VACUUM INTO creates its file with the default
|
||||||
|
// mode: made there directly, a full copy of the store — password hashes,
|
||||||
|
// the engine-key secret — sat readable by any local user for as long as
|
||||||
|
// the backup took.
|
||||||
|
tmpDir, err := os.MkdirTemp(scratch, ".store-")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tmpDir)
|
||||||
|
tmpPath := filepath.Join(tmpDir, "store")
|
||||||
|
|
||||||
|
name := "store.sqlite"
|
||||||
|
if store.IsPostgres(addr) {
|
||||||
|
name = "store.pgdump"
|
||||||
|
if err := pgDump(ctx, addr, tmpPath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
db, err := store.Open(ctx, addr)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
err = db.SnapshotTo(ctx, tmpPath)
|
||||||
|
_ = db.Close()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return addFile(tw, tmpPath, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// pgDump runs pg_dump in its custom format, which pg_restore can restore
|
||||||
|
// selectively. The connection is given through the environment, not the
|
||||||
|
// command line: a DSN carries the password, and every user on the machine
|
||||||
|
// can read another process's arguments.
|
||||||
|
func pgDump(ctx context.Context, dsn, out string) error {
|
||||||
|
u, err := url.Parse(dsn)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the store address is not a URL")
|
||||||
|
}
|
||||||
|
env := os.Environ()
|
||||||
|
set := func(k, v string) {
|
||||||
|
if v != "" {
|
||||||
|
env = append(env, k+"="+v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
set("PGHOST", u.Hostname())
|
||||||
|
set("PGPORT", u.Port())
|
||||||
|
set("PGUSER", u.User.Username())
|
||||||
|
if pw, ok := u.User.Password(); ok {
|
||||||
|
set("PGPASSWORD", pw)
|
||||||
|
}
|
||||||
|
set("PGDATABASE", strings.TrimPrefix(u.Path, "/"))
|
||||||
|
set("PGSSLMODE", u.Query().Get("sslmode"))
|
||||||
|
cmd := exec.CommandContext(ctx, "pg_dump", "--format=custom", "--no-owner", "--file="+out) //nolint:gosec // fixed binary, our own file
|
||||||
|
cmd.Env = env
|
||||||
|
if b, err := cmd.CombinedOutput(); err != nil {
|
||||||
|
return fmt.Errorf("pg_dump: %v: %s", err, strings.TrimSpace(string(b)))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func addTree(tw *tar.Writer, root, as string) error {
|
||||||
|
return filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rel, _ := filepath.Rel(root, p)
|
||||||
|
name := filepath.ToSlash(filepath.Join(as, rel))
|
||||||
|
info, err := d.Info()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case d.IsDir():
|
||||||
|
return tw.WriteHeader(&tar.Header{Name: name + "/", Mode: int64(info.Mode().Perm()),
|
||||||
|
Typeflag: tar.TypeDir, ModTime: info.ModTime()})
|
||||||
|
case info.Mode().IsRegular():
|
||||||
|
return addFile(tw, p, name)
|
||||||
|
}
|
||||||
|
return nil // sockets, links: not configuration
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func addFile(tw *tar.Writer, path, name string) error {
|
||||||
|
f, err := os.Open(path) //nolint:gosec // files we chose to back up
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
info, err := f.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: int64(info.Mode().Perm()),
|
||||||
|
Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = io.Copy(tw, f)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func addBytes(tw *tar.Writer, name string, b []byte) error {
|
||||||
|
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b)),
|
||||||
|
ModTime: time.Now(), Typeflag: tar.TypeReg}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err := tw.Write(b)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// pruneBackups keeps the newest keep archives. Only files this command
|
||||||
|
// wrote are considered, so nothing else in the directory is ever removed.
|
||||||
|
func pruneBackups(dir string, keep int) (int, error) {
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for _, e := range entries {
|
||||||
|
if n := e.Name(); strings.HasPrefix(n, backupPrefix) && strings.HasSuffix(n, ".tar.gz") {
|
||||||
|
names = append(names, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The name carries a UTC timestamp, so name order is age order.
|
||||||
|
slices.Sort(names)
|
||||||
|
removed := 0
|
||||||
|
for len(names) > max(keep, 1) {
|
||||||
|
if err := os.Remove(filepath.Join(dir, names[0])); err != nil {
|
||||||
|
return removed, err
|
||||||
|
}
|
||||||
|
names = names[1:]
|
||||||
|
removed++
|
||||||
|
}
|
||||||
|
return removed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewestBackup is the modification time of the newest complete archive in
|
||||||
|
// dir, or zero when there is none.
|
||||||
|
func NewestBackup(dir string) time.Time {
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
var newest time.Time
|
||||||
|
for _, e := range entries {
|
||||||
|
if n := e.Name(); !strings.HasPrefix(n, backupPrefix) || !strings.HasSuffix(n, ".tar.gz") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if info, err := e.Info(); err == nil && info.ModTime().After(newest) {
|
||||||
|
newest = info.ModTime()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return newest
|
||||||
|
}
|
||||||
|
|
||||||
|
// unlistable says why the control plane, which does not run as root, could
|
||||||
|
// not see the archives in dir, or "" when it can.
|
||||||
|
func unlistable(dir string) string {
|
||||||
|
info, err := os.Stat(dir)
|
||||||
|
if err != nil {
|
||||||
|
return err.Error()
|
||||||
|
}
|
||||||
|
if info.Mode().Perm()&0o005 != 0o005 {
|
||||||
|
return fmt.Sprintf("%s is %v, so only its owner can list it", dir, info.Mode().Perm())
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"compress/gzip"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestABackupHoldsTheStoreAndTheConfiguration(t *testing.T) {
|
||||||
|
tmp := t.TempDir()
|
||||||
|
dbPath := filepath.Join(tmp, "store.db")
|
||||||
|
db, err := store.Open(t.Context(), dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
secret, err := db.ClusterSecret(t.Context(), "engine-key")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = db.Close()
|
||||||
|
|
||||||
|
etc := filepath.Join(tmp, "etc")
|
||||||
|
_ = os.MkdirAll(filepath.Join(etc, "pki"), 0o700)
|
||||||
|
_ = os.WriteFile(filepath.Join(etc, "server.yaml"), []byte("listen: :8080\n"), 0o644)
|
||||||
|
_ = os.WriteFile(filepath.Join(etc, "pki", "ca-key.pem"), []byte("KEY"), 0o600)
|
||||||
|
|
||||||
|
out := filepath.Join(tmp, "backups")
|
||||||
|
path, err := writeBackup(t.Context(), dbPath, etc, out)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info, _ := os.Stat(path)
|
||||||
|
if info.Mode().Perm() != 0o600 {
|
||||||
|
t.Errorf("the archive holds the CA's key and is %v", info.Mode().Perm())
|
||||||
|
}
|
||||||
|
|
||||||
|
files := map[string][]byte{}
|
||||||
|
f, _ := os.Open(path)
|
||||||
|
defer f.Close()
|
||||||
|
gz, err := gzip.NewReader(f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
tr := tar.NewReader(gz)
|
||||||
|
for {
|
||||||
|
h, err := tr.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b, _ := io.ReadAll(tr)
|
||||||
|
files[h.Name] = b
|
||||||
|
}
|
||||||
|
for _, want := range []string{"MANIFEST", "store.sqlite", "etc/llmgw/server.yaml", "etc/llmgw/pki/ca-key.pem"} {
|
||||||
|
if _, ok := files[want]; !ok {
|
||||||
|
t.Errorf("%s is not in the archive", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the store in it is a working store, with what was in it.
|
||||||
|
restored := filepath.Join(tmp, "restored.db")
|
||||||
|
_ = os.WriteFile(restored, files["store.sqlite"], 0o600)
|
||||||
|
rdb, err := store.Open(t.Context(), restored)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("the backed-up store does not open: %v", err)
|
||||||
|
}
|
||||||
|
defer rdb.Close()
|
||||||
|
if got, _ := rdb.ClusterSecret(t.Context(), "engine-key"); got != secret {
|
||||||
|
t.Error("the restored store lost the engine key secret; every engine would be locked out")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOldBackupsArePrunedAndTheNewestIsFound(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for i, stamp := range []string{"20260101T000000Z", "20260102T000000Z", "20260103T000000Z"} {
|
||||||
|
p := filepath.Join(dir, backupPrefix+"h-"+stamp+".tar.gz")
|
||||||
|
_ = os.WriteFile(p, []byte("x"), 0o600)
|
||||||
|
mt := time.Date(2026, 1, 1+i, 0, 0, 0, 0, time.UTC)
|
||||||
|
_ = os.Chtimes(p, mt, mt)
|
||||||
|
}
|
||||||
|
_ = os.WriteFile(filepath.Join(dir, "someone-elses-file"), []byte("x"), 0o600)
|
||||||
|
_ = os.WriteFile(filepath.Join(dir, backupPrefix+"h-20260104T000000Z.tar.gz.part"), []byte("x"), 0o600)
|
||||||
|
|
||||||
|
removed, err := pruneBackups(dir, 2)
|
||||||
|
if err != nil || removed != 1 {
|
||||||
|
t.Fatalf("removed %d, %v", removed, err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(dir, "someone-elses-file")); err != nil {
|
||||||
|
t.Error("a file the backup did not write was removed")
|
||||||
|
}
|
||||||
|
newest := NewestBackup(dir)
|
||||||
|
if !newest.Equal(time.Date(2026, 1, 3, 0, 0, 0, 0, time.UTC)) {
|
||||||
|
t.Errorf("newest %v; an interrupted .part must not count", newest)
|
||||||
|
}
|
||||||
|
if !NewestBackup(filepath.Join(dir, "none")).IsZero() {
|
||||||
|
t.Error("a missing directory reported a backup")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestABackupDirectoryTheControlPlaneCannotListIsSaid(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
_ = os.Chmod(dir, 0o700)
|
||||||
|
if unlistable(dir) == "" {
|
||||||
|
t.Error("a directory only its owner can list was taken as visible")
|
||||||
|
}
|
||||||
|
_ = os.Chmod(dir, 0o755)
|
||||||
|
if why := unlistable(dir); why != "" {
|
||||||
|
t.Errorf("a listable directory: %s", why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoScratchCopyOfTheStoreIsLeftBehind(t *testing.T) {
|
||||||
|
// The snapshot is made before it is archived. Made straight into the
|
||||||
|
// listable archive directory, it was readable by anyone for that time.
|
||||||
|
tmp := t.TempDir()
|
||||||
|
dbPath := filepath.Join(tmp, "store.db")
|
||||||
|
db, err := store.Open(t.Context(), dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = db.Close()
|
||||||
|
out := filepath.Join(tmp, "backups")
|
||||||
|
if _, err := writeBackup(t.Context(), dbPath, t.TempDir(), out); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entries, _ := os.ReadDir(out)
|
||||||
|
for _, e := range entries {
|
||||||
|
if strings.HasPrefix(e.Name(), ".store") {
|
||||||
|
t.Errorf("a scratch copy of the store was left behind: %s", e.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -41,6 +41,11 @@ func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
|||||||
cfg.MetricsURL = metrics
|
cfg.MetricsURL = metrics
|
||||||
cfg.Engine = engine
|
cfg.Engine = engine
|
||||||
cfg.APIKey = key
|
cfg.APIKey = key
|
||||||
|
// A replica's engine demands its own key. Asked for rather than
|
||||||
|
// typed: the admin API knows which replica answers at the target.
|
||||||
|
if cfg.APIKey == "" {
|
||||||
|
cfg.APIKey = replicaKey(cmd, client, target)
|
||||||
|
}
|
||||||
if len(steps) > 0 {
|
if len(steps) > 0 {
|
||||||
cfg.Concurrency = steps
|
cfg.Concurrency = steps
|
||||||
}
|
}
|
||||||
@@ -227,3 +232,18 @@ func ms(d time.Duration) string {
|
|||||||
return d.Round(10 * time.Millisecond).String()
|
return d.Round(10 * time.Millisecond).String()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// replicaKey is the engine key of the replica at target, or "" when target is
|
||||||
|
// not a replica (the gateway, say) or the admin API cannot be asked.
|
||||||
|
func replicaKey(cmd *cobra.Command, client func() *ctlClient, target string) string {
|
||||||
|
var reps []adminapi.ReplicaView
|
||||||
|
if err := client().get(cmd.Context(), "/api/v1/replicas", &reps); err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
for _, rep := range reps {
|
||||||
|
if sameEndpoint(rep.Endpoint, target) {
|
||||||
|
return rep.EngineKey
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
+57
-7
@@ -67,7 +67,9 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if body != nil {
|
// On every change, body or not: the admin listener refuses anything else,
|
||||||
|
// because a browser can send a bodiless POST across origins unasked.
|
||||||
|
if method != http.MethodGet {
|
||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", "application/json")
|
||||||
}
|
}
|
||||||
// Who made the change, kept with a stored model so a catalog that grew
|
// Who made the change, kept with a stored model so a catalog that grew
|
||||||
@@ -81,7 +83,10 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any)
|
|||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
// Any 2xx. The API answers a change it has queued rather than made — a
|
||||||
|
// restart — with 202, and treating that as a failure printed
|
||||||
|
// "Error: 202 Accepted" over a request that had worked.
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||||
var e struct {
|
var e struct {
|
||||||
Error string `json:"error"`
|
Error string `json:"error"`
|
||||||
}
|
}
|
||||||
@@ -117,6 +122,7 @@ func newCtlCmd() *cobra.Command {
|
|||||||
cmd.AddCommand(
|
cmd.AddCommand(
|
||||||
newNodesCmd(client, &asJSON),
|
newNodesCmd(client, &asJSON),
|
||||||
newLogsCmd(client, &asJSON),
|
newLogsCmd(client, &asJSON),
|
||||||
|
newRequestCmd(client, &asJSON),
|
||||||
newModelsCmd(client, &asJSON),
|
newModelsCmd(client, &asJSON),
|
||||||
newReplicasCmd(client, &asJSON),
|
newReplicasCmd(client, &asJSON),
|
||||||
newExplainCmd(client, &asJSON),
|
newExplainCmd(client, &asJSON),
|
||||||
@@ -131,12 +137,16 @@ func newCtlCmd() *cobra.Command {
|
|||||||
// apply to it; it lives here because this is where operators look.
|
// apply to it; it lives here because this is where operators look.
|
||||||
newBenchCmd(client, &asJSON),
|
newBenchCmd(client, &asJSON),
|
||||||
newStatusCmd(client, &asJSON),
|
newStatusCmd(client, &asJSON),
|
||||||
newModelCmd(client),
|
newModelCmd(client, &asJSON),
|
||||||
newScaleCmd(client),
|
newScaleCmd(client),
|
||||||
|
newRestartCmd(client),
|
||||||
|
newRetryCmd(client),
|
||||||
|
newUpgradeCmd(client),
|
||||||
newEngineCheckCmd(),
|
newEngineCheckCmd(),
|
||||||
newToolCheckCmd(),
|
newToolCheckCmd(),
|
||||||
newWeightsCmd(client, &asJSON),
|
newWeightsCmd(client, &asJSON),
|
||||||
newCapacityCmd(client, &asJSON),
|
newCapacityCmd(client, &asJSON),
|
||||||
|
newQuotaCmd(client, &asJSON),
|
||||||
newUsageCmd(client, &asJSON),
|
newUsageCmd(client, &asJSON),
|
||||||
newCertCmd(),
|
newCertCmd(),
|
||||||
)
|
)
|
||||||
@@ -242,6 +252,14 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
|||||||
// actually served, never what is merely catalogued.
|
// actually served, never what is merely catalogued.
|
||||||
func modelStatus(m adminapi.ModelView) string {
|
func modelStatus(m adminapi.ModelView) string {
|
||||||
switch {
|
switch {
|
||||||
|
case m.Parked:
|
||||||
|
return "parked"
|
||||||
|
case m.External != "":
|
||||||
|
return "external: " + m.External
|
||||||
|
case m.HeldFor != "":
|
||||||
|
return "gave room to " + m.HeldFor
|
||||||
|
case m.Unplaceable:
|
||||||
|
return "cannot place"
|
||||||
case m.Min == 0 && m.Total == 0:
|
case m.Min == 0 && m.Total == 0:
|
||||||
return "stopped"
|
return "stopped"
|
||||||
case m.Ready >= m.Min && m.Min > 0:
|
case m.Ready >= m.Min && m.Min > 0:
|
||||||
@@ -310,17 +328,27 @@ func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
|||||||
}
|
}
|
||||||
|
|
||||||
w := tab()
|
w := tab()
|
||||||
fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tALIASES\tPROFILES\tFALLBACKS")
|
fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tCLASS\tRESERVES\tALIASES\tPROFILES\tFALLBACKS")
|
||||||
|
var warnings []string
|
||||||
for _, m := range models {
|
for _, m := range models {
|
||||||
serving := "no"
|
serving := "no"
|
||||||
if m.Ready > 0 {
|
if m.Ready > 0 {
|
||||||
serving = "yes"
|
serving = "yes"
|
||||||
}
|
}
|
||||||
fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\n",
|
fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||||
m.Name, serving, m.Ready, m.Min, modelStatus(m),
|
m.Name, serving, m.Ready, m.Min, modelStatus(m), m.Class, reserves(m),
|
||||||
join(m.Aliases), join(m.Profiles), join(m.Fallbacks))
|
join(m.Aliases), join(m.Profiles), join(m.Fallbacks))
|
||||||
|
for _, x := range m.Warnings {
|
||||||
|
warnings = append(warnings, m.Name+": "+x)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return w.Flush()
|
if err := w.Flush(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, x := range warnings {
|
||||||
|
fmt.Println("warning:", x)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -368,6 +396,9 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
|||||||
if r.Stale {
|
if r.Stale {
|
||||||
state += " (stale profile)"
|
state += " (stale profile)"
|
||||||
}
|
}
|
||||||
|
if r.Restart != "" {
|
||||||
|
state += " (restart " + r.Restart + ")"
|
||||||
|
}
|
||||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||||
r.ID, r.Model, state, r.Pool, join(nodes),
|
r.ID, r.Model, state, r.Pool, join(nodes),
|
||||||
(time.Duration(r.AgeSec) * time.Second).String(),
|
(time.Duration(r.AgeSec) * time.Second).String(),
|
||||||
@@ -495,3 +526,22 @@ func dash(s string) string {
|
|||||||
}
|
}
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reserves is what one replica of a model holds of a node, with what it was
|
||||||
|
// measured using beside it: "6G 2c (5.1G used)".
|
||||||
|
func reserves(m adminapi.ModelView) string {
|
||||||
|
if m.MemoryGB == 0 && m.CPUCores == 0 {
|
||||||
|
return "-"
|
||||||
|
}
|
||||||
|
out := fmt.Sprintf("%dG", m.MemoryGB)
|
||||||
|
if m.CPUCores > 0 {
|
||||||
|
out += fmt.Sprintf(" %dc", m.CPUCores)
|
||||||
|
}
|
||||||
|
if m.MeasuredMemoryGB > 0 {
|
||||||
|
out += fmt.Sprintf(" (%.1fG used)", m.MeasuredMemoryGB)
|
||||||
|
}
|
||||||
|
if len(m.Warnings) > 0 {
|
||||||
|
out += " !"
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "request ID",
|
||||||
|
Short: "What one request went through: replica, queue, first token, tokens",
|
||||||
|
Long: "Every response carries its id in X-Request-Id, and a client may send its\n" +
|
||||||
|
"own. Given one, this says which replica answered, how long admission held\n" +
|
||||||
|
"it, how long the first token took and what it cost — the answer to \"why\n" +
|
||||||
|
"was this slow\". The access log has the same fields under \"request\".",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
var v adminapi.RequestView
|
||||||
|
if err := client().get(cmd.Context(), "/api/v1/requests/"+url.PathEscape(args[0]), &v); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
|
||||||
|
}
|
||||||
|
model := v.Model
|
||||||
|
if v.ServedModel != "" && v.ServedModel != v.Model {
|
||||||
|
model += " (served by " + v.ServedModel + ")"
|
||||||
|
}
|
||||||
|
out := cmd.OutOrStdout()
|
||||||
|
fmt.Fprintf(out, "request %s at %s\n", v.RequestID, v.At.UTC().Format("2006-01-02 15:04:05Z"))
|
||||||
|
fmt.Fprintf(out, "model %s\n", model)
|
||||||
|
fmt.Fprintf(out, "replica %s\n", or(v.Replica, "- (none answered)"))
|
||||||
|
fmt.Fprintf(out, "status %d\n", v.Status)
|
||||||
|
fmt.Fprintf(out, "time %d ms total: %d queued, %d to the first token\n", v.LatencyMs, v.QueueMs, v.TTFTMs)
|
||||||
|
fmt.Fprintf(out, "tokens %d in, %d out\n", v.PromptTokens, v.CompletionTokens)
|
||||||
|
fmt.Fprintf(out, "who key %s, project %s%s\n", v.KeyID, v.ProjectID, prefixed(", user ", v.UserID))
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newRestartCmd(client func() *ctlClient) *cobra.Command {
|
||||||
|
var (
|
||||||
|
models []string
|
||||||
|
all bool
|
||||||
|
)
|
||||||
|
c := &cobra.Command{
|
||||||
|
Use: "restart [REPLICA...]",
|
||||||
|
Short: "Replace replicas with new ones, starting each replacement before stopping the old",
|
||||||
|
Long: "How a running engine picks up what only applies at start: an engine key,\n" +
|
||||||
|
"an edited profile, a new engine version.\n\n" +
|
||||||
|
"Each replacement is placed beside the replica it replaces, and the old one\n" +
|
||||||
|
"keeps serving until the new one is ready. Replicas of one model are\n" +
|
||||||
|
"replaced one at a time. Where nothing else has room for a second copy —\n" +
|
||||||
|
"the usual case on a small CPU fleet — the old one is stopped after the\n" +
|
||||||
|
"drain grace period (5 minutes by default) and the new one starts in its\n" +
|
||||||
|
"place; a model with only that replica is unserved while it loads.\n\n" +
|
||||||
|
" llmgw ctl restart qwen2.5-7b-3f2a91c0\n" +
|
||||||
|
" llmgw ctl restart --model qwen2.5-7b\n" +
|
||||||
|
" llmgw ctl restart --all",
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
if len(args) == 0 && len(models) == 0 && !all {
|
||||||
|
return errors.New("name replicas, --model, or --all")
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(adminapi.RestartRequest{Replicas: args, Models: models, All: all})
|
||||||
|
var res adminapi.RestartResult
|
||||||
|
if err := client().post(cmd.Context(), "/api/v1/restart", body, &res); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%d replica(s) queued for restart; watch with: llmgw ctl replicas\n", len(res.Queued))
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c.Flags().StringArrayVar(&models, "model", nil, "every replica of this model; repeatable")
|
||||||
|
c.Flags().BoolVar(&all, "all", false, "every ready replica in the cluster")
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRetryCmd(client func() *ctlClient) *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "retry MODEL",
|
||||||
|
Short: "Try a parked model again, and clear its backoff",
|
||||||
|
Long: "A model whose replicas fail every time they start is parked after five\n" +
|
||||||
|
"failures in a row: it is not placed again until its definition changes\n" +
|
||||||
|
"or this is run. Run it once the cause — the weights, the engine, the\n" +
|
||||||
|
"profile — has been dealt with; `llmgw ctl status` lists parked models with\n" +
|
||||||
|
"their last error.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
var res struct {
|
||||||
|
WasParked bool `json:"was_parked"`
|
||||||
|
}
|
||||||
|
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/retry", []byte(`{}`), &res); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if res.WasParked {
|
||||||
|
fmt.Printf("%s unparked; it is placed on the next pass\n", args[0])
|
||||||
|
} else {
|
||||||
|
fmt.Printf("%s was not parked; its backoff is cleared\n", args[0])
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAQueuedChangeIsNotReportedAsAnError(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusAccepted)
|
||||||
|
_, _ = w.Write([]byte(`{"queued":["r1"]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
var out struct {
|
||||||
|
Queued []string `json:"queued"`
|
||||||
|
}
|
||||||
|
if err := newCtlClient(srv.URL).post(t.Context(), "/api/v1/restart", []byte(`{}`), &out); err != nil {
|
||||||
|
t.Fatalf("202 read as a failure: %v", err)
|
||||||
|
}
|
||||||
|
if len(out.Queued) != 1 {
|
||||||
|
t.Errorf("the body of a 202 was not read: %+v", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,201 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"runtime"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newUpgradeCmd(client func() *ctlClient) *cobra.Command {
|
||||||
|
var (
|
||||||
|
nodes []string
|
||||||
|
skipCP bool
|
||||||
|
releases string
|
||||||
|
perNode time.Duration
|
||||||
|
serverSvc string
|
||||||
|
)
|
||||||
|
c := &cobra.Command{
|
||||||
|
Use: "upgrade VERSION",
|
||||||
|
Short: "Upgrade this control plane, then every node, one at a time",
|
||||||
|
Long: "Moves the cluster to a release: v0.10.4, or a candidate like v0.10.4-rc.2.\n\n" +
|
||||||
|
"This machine first — run it with sudo on the control plane, since it\n" +
|
||||||
|
"replaces /usr/local/bin/llmgw and restarts llmgw-server. Then each node, in\n" +
|
||||||
|
"turn: its agent fetches the release for its own platform, checks it\n" +
|
||||||
|
"against the release's checksums, swaps it in and restarts. Engines keep\n" +
|
||||||
|
"serving through all of it and are adopted again; no ssh into the nodes is\n" +
|
||||||
|
"needed.\n\n" +
|
||||||
|
"A node counts as done when it is back on the new version, ready, with as\n" +
|
||||||
|
"many ready replicas as before. One that is not within --timeout stops the\n" +
|
||||||
|
"upgrade there, so a bad release reaches one node rather than all of them.\n" +
|
||||||
|
"Nodes already on the version are skipped, so running it again resumes.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
v := args[0]
|
||||||
|
if !release.Tag.MatchString(v) {
|
||||||
|
return fmt.Errorf("%q is not a release version (v1.2.3 or v1.2.3-rc.4)", v)
|
||||||
|
}
|
||||||
|
out := cmd.OutOrStdout()
|
||||||
|
ctx := cmd.Context()
|
||||||
|
if !skipCP {
|
||||||
|
if err := upgradeControlPlane(ctx, out, client, v, releases, serverSvc); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return upgradeNodes(ctx, out, client, v, nodes, perNode)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c.Flags().StringSliceVar(&nodes, "nodes", nil, "only these nodes (default: all)")
|
||||||
|
c.Flags().BoolVar(&skipCP, "skip-control-plane", false, "leave this machine as it is and upgrade only nodes")
|
||||||
|
c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from, for this machine")
|
||||||
|
c.Flags().DurationVar(&perNode, "timeout", 3*time.Minute, "how long a node may take to come back")
|
||||||
|
c.Flags().StringVar(&serverSvc, "service", "llmgw-server", "this machine's systemd unit")
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func upgradeControlPlane(ctx context.Context, out io.Writer, client func() *ctlClient, v, releases, svc string) error {
|
||||||
|
if version.Version == v {
|
||||||
|
fmt.Fprintf(out, "this machine already runs %s\n", v)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if os.Geteuid() != 0 {
|
||||||
|
return errors.New("replacing this machine's binary needs root: run it with sudo, " +
|
||||||
|
"or pass --skip-control-plane to upgrade only the nodes")
|
||||||
|
}
|
||||||
|
self, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "== this machine: %s -> %s\n", version.Version, v)
|
||||||
|
backup, err := release.Install(ctx, nil, releases, v, runtime.GOOS, runtime.GOARCH, self)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, " installed, checksum verified; the old binary is %s\n", backup)
|
||||||
|
var before []adminapi.NodeView
|
||||||
|
_ = client().get(ctx, "/api/v1/nodes", &before)
|
||||||
|
if b, err := exec.CommandContext(ctx, "systemctl", "restart", svc).CombinedOutput(); err != nil { //nolint:gosec // fixed command
|
||||||
|
return fmt.Errorf("restarting %s: %v: %s", svc, err, strings.TrimSpace(string(b)))
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, " %s restarted; waiting for its nodes to reconnect\n", svc)
|
||||||
|
// Back when the admin API answers and every node that was ready is ready
|
||||||
|
// again: agents reconnect within seconds of the control plane returning.
|
||||||
|
want := 0
|
||||||
|
for _, n := range before {
|
||||||
|
if n.State == "ready" {
|
||||||
|
want++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(2 * time.Minute)
|
||||||
|
for {
|
||||||
|
var now []adminapi.NodeView
|
||||||
|
if err := client().get(ctx, "/api/v1/nodes", &now); err == nil {
|
||||||
|
ready := 0
|
||||||
|
for _, n := range now {
|
||||||
|
if n.State == "ready" {
|
||||||
|
ready++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ready >= want {
|
||||||
|
fmt.Fprintf(out, " back: %d of %d nodes ready\n", ready, len(now))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
return fmt.Errorf("%s did not come back with its %d nodes within two minutes; "+
|
||||||
|
"see journalctl -u %s, and %s holds the previous binary", svc, want, svc, backup)
|
||||||
|
}
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient, v string, only []string, perNode time.Duration) error {
|
||||||
|
var nodes []adminapi.NodeView
|
||||||
|
if err := client().get(ctx, "/api/v1/nodes", &nodes); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
slices.SortFunc(nodes, func(a, b adminapi.NodeView) int { return strings.Compare(a.ID, b.ID) })
|
||||||
|
done, skipped := 0, 0
|
||||||
|
for _, n := range nodes {
|
||||||
|
if len(only) > 0 && !slices.Contains(only, n.ID) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if n.AgentVersion == v {
|
||||||
|
skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
start := time.Now()
|
||||||
|
before := readyReplicasOn(ctx, client, n.ID)
|
||||||
|
fmt.Fprintf(out, "== %s: %s -> %s\n", n.ID, n.AgentVersion, v)
|
||||||
|
body, _ := json.Marshal(adminapi.UpgradeRequest{Version: v})
|
||||||
|
if err := client().post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil {
|
||||||
|
return fmt.Errorf("%s: %w\nstopped here; `llmgw ctl logs %s` says more", n.ID, err, n.ID)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, " new binary in place; waiting for it to come back with its %d replica(s)\n", before)
|
||||||
|
deadline := time.Now().Add(perNode)
|
||||||
|
for {
|
||||||
|
var now []adminapi.NodeView
|
||||||
|
_ = client().get(ctx, "/api/v1/nodes", &now)
|
||||||
|
var cur adminapi.NodeView
|
||||||
|
for _, x := range now {
|
||||||
|
if x.ID == n.ID {
|
||||||
|
cur = x
|
||||||
|
}
|
||||||
|
}
|
||||||
|
after := readyReplicasOn(ctx, client, n.ID)
|
||||||
|
if cur.AgentVersion == v && cur.State == "ready" && after >= before {
|
||||||
|
fmt.Fprintf(out, " done in %s: ready, replicas %d/%d\n",
|
||||||
|
time.Since(start).Round(time.Second), after, before)
|
||||||
|
done++
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
return fmt.Errorf("%s did not come back within %s (version %q, state %q, replicas %d/%d)\n"+
|
||||||
|
"stopped here, before the next node; `llmgw ctl logs %s` says why, and the node "+
|
||||||
|
"keeps its previous binary beside the new one", n.ID, perNode, cur.AgentVersion,
|
||||||
|
cur.State, after, before, n.ID)
|
||||||
|
}
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "%d node(s) upgraded to %s, %d already on it\n", done, v, skipped)
|
||||||
|
// The one machine this does not reach. Said every time, because it is
|
||||||
|
// exactly the kind of thing that stays three versions behind unnoticed.
|
||||||
|
fmt.Fprintf(out, "The model mirror's host is not upgraded by this; on it, run:\n"+
|
||||||
|
" sudo llmgw upgrade %s --service llmgw-mirror\n", v)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readyReplicasOn counts the ready replicas with a shard on a node.
|
||||||
|
func readyReplicasOn(ctx context.Context, client func() *ctlClient, node string) int {
|
||||||
|
var reps []adminapi.ReplicaView
|
||||||
|
if err := client().get(ctx, "/api/v1/replicas", &reps); err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
n := 0
|
||||||
|
for _, r := range reps {
|
||||||
|
if r.State != "ready" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, s := range r.Shards {
|
||||||
|
if s.Node == node {
|
||||||
|
n++
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeFleet is an admin API whose nodes come back on the version they are told
|
||||||
|
// to upgrade to — except those listed in stuck.
|
||||||
|
type fakeFleet struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
versions map[string]string
|
||||||
|
stuck map[string]bool
|
||||||
|
told []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeFleet) server(t *testing.T) *httptest.Server {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
switch {
|
||||||
|
case r.URL.Path == "/api/v1/nodes":
|
||||||
|
var out []adminapi.NodeView
|
||||||
|
for id, v := range f.versions {
|
||||||
|
out = append(out, adminapi.NodeView{ID: id, State: "ready", AgentVersion: v})
|
||||||
|
}
|
||||||
|
_ = json.NewEncoder(w).Encode(out)
|
||||||
|
case r.URL.Path == "/api/v1/replicas":
|
||||||
|
var out []adminapi.ReplicaView
|
||||||
|
for id := range f.versions {
|
||||||
|
out = append(out, adminapi.ReplicaView{ID: "r-" + id, State: "ready",
|
||||||
|
Shards: []adminapi.ShardView{{Node: id}}})
|
||||||
|
}
|
||||||
|
_ = json.NewEncoder(w).Encode(out)
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/upgrade"):
|
||||||
|
id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/nodes/"), "/upgrade")
|
||||||
|
var req adminapi.UpgradeRequest
|
||||||
|
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||||
|
f.told = append(f.told, id)
|
||||||
|
if !f.stuck[id] {
|
||||||
|
f.versions[id] = req.Version
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{}`))
|
||||||
|
default:
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnUpgradeGoesNodeByNodeAndSkipsTheDone(t *testing.T) {
|
||||||
|
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.1", "c": "v1.0.0"}}
|
||||||
|
srv := f.server(t)
|
||||||
|
var out bytes.Buffer
|
||||||
|
err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 5*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Join(f.told, ",") != "a,c" {
|
||||||
|
t.Errorf("told %v; b was already on the version", f.told)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "2 node(s) upgraded to v1.0.1, 1 already on it") {
|
||||||
|
t.Errorf("output:\n%s", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnUpgradeStopsAtTheFirstNodeThatDoesNotComeBack(t *testing.T) {
|
||||||
|
// A bad release reaches one node, not all of them.
|
||||||
|
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.0"}, stuck: map[string]bool{"a": true}}
|
||||||
|
srv := f.server(t)
|
||||||
|
var out bytes.Buffer
|
||||||
|
err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 3*time.Second)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "a did not come back") {
|
||||||
|
t.Fatalf("err %v", err)
|
||||||
|
}
|
||||||
|
if len(f.told) != 1 {
|
||||||
|
t.Errorf("went on to %v after a node failed to come back", f.told)
|
||||||
|
}
|
||||||
|
}
|
||||||
+56
-14
@@ -8,6 +8,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -232,7 +233,8 @@ func newMirrorRemoveCmd(dir *string) *cobra.Command {
|
|||||||
func newMirrorServeCmd(dir *string) *cobra.Command {
|
func newMirrorServeCmd(dir *string) *cobra.Command {
|
||||||
var listen, apiListen, base string
|
var listen, apiListen, base string
|
||||||
var tlsc tlsconf.Config
|
var tlsc tlsconf.Config
|
||||||
var allow []string
|
var allow, clients []string
|
||||||
|
var allowPrivate bool
|
||||||
|
|
||||||
c := &cobra.Command{
|
c := &cobra.Command{
|
||||||
Use: "serve",
|
Use: "serve",
|
||||||
@@ -258,10 +260,16 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
|
|||||||
if err := tlsc.Validate(); err != nil {
|
if err := tlsc.Validate(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if len(clients) > 0 && !tlsc.Enabled() {
|
||||||
|
return fmt.Errorf("--api-client names a certificate, and needs --tls-ca, --tls-cert and --tls-key")
|
||||||
|
}
|
||||||
|
|
||||||
out := cmd.OutOrStdout()
|
out := cmd.OutOrStdout()
|
||||||
log := slog.Default()
|
log := slog.Default()
|
||||||
mgr := mirror.NewManager(*dir, log)
|
mgr := mirror.NewManager(*dir, log)
|
||||||
|
if allowPrivate {
|
||||||
|
mgr.AllowPrivateSources()
|
||||||
|
}
|
||||||
api := mirror.NewAPI(mgr, strings.TrimSuffix(base, "/"))
|
api := mirror.NewAPI(mgr, strings.TrimSuffix(base, "/"))
|
||||||
|
|
||||||
// http.Dir refuses paths that escape the root, which is the whole
|
// http.Dir refuses paths that escape the root, which is the whole
|
||||||
@@ -278,7 +286,7 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
|
|||||||
}
|
}
|
||||||
admin := &http.Server{
|
admin := &http.Server{
|
||||||
Addr: apiListen,
|
Addr: apiListen,
|
||||||
Handler: onlyFrom(allowed, api.Handler()),
|
Handler: onlyClients(clients, apiGate(tlsc.Enabled(), allowed, api.Handler())),
|
||||||
ReadHeaderTimeout: 10 * time.Second,
|
ReadHeaderTimeout: 10 * time.Second,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -290,14 +298,20 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
|
|||||||
switch {
|
switch {
|
||||||
case tlsc.Enabled():
|
case tlsc.Enabled():
|
||||||
fmt.Fprintf(out, "management API on %s, mutual TLS against %s\n", apiListen, tlsc.CA)
|
fmt.Fprintf(out, "management API on %s, mutual TLS against %s\n", apiListen, tlsc.CA)
|
||||||
|
if len(clients) == 0 {
|
||||||
|
// The same root signs every node's certificate, so without a
|
||||||
|
// name any node could delete models here.
|
||||||
|
log.Warn("the management API takes any certificate from " + tlsc.CA +
|
||||||
|
", every node's included; name the control plane's with --api-client")
|
||||||
|
}
|
||||||
case len(allowed) > 0:
|
case len(allowed) > 0:
|
||||||
fmt.Fprintf(out, "management API on %s, for %s and this machine only\n", apiListen, strings.Join(allow, ", "))
|
fmt.Fprintf(out, "management API on %s, for %s and this machine only\n", apiListen, strings.Join(allow, ", "))
|
||||||
default:
|
default:
|
||||||
// Said plainly rather than left to be discovered. An API that
|
// This machine only, which was once "anyone": an API that can
|
||||||
// can delete models and start downloads, open to anyone who can
|
// delete models and start downloads, open to every node that
|
||||||
// reach the port, is a choice the operator should have made
|
// could reach the port, is a choice the operator should make
|
||||||
// rather than inherited from a default.
|
// with --api-allow rather than inherit.
|
||||||
fmt.Fprintf(out, "management API on %s, UNAUTHENTICATED — keep it on a trusted network\n", apiListen)
|
fmt.Fprintf(out, "management API on %s, for this machine only (--api-allow to add the control plane)\n", apiListen)
|
||||||
}
|
}
|
||||||
fmt.Fprintf(out, "catalog sources should point at %s\n", base)
|
fmt.Fprintf(out, "catalog sources should point at %s\n", base)
|
||||||
|
|
||||||
@@ -312,7 +326,11 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
|
|||||||
c.Flags().StringVar(&base, "base", "http://localhost:8081",
|
c.Flags().StringVar(&base, "base", "http://localhost:8081",
|
||||||
"the address nodes use to reach this mirror, as it should appear in catalog sources")
|
"the address nodes use to reach this mirror, as it should appear in catalog sources")
|
||||||
c.Flags().StringArrayVar(&allow, "api-allow", nil,
|
c.Flags().StringArrayVar(&allow, "api-allow", nil,
|
||||||
"an address or network the management API answers, besides this machine; repeatable (default: anyone)")
|
"an address or network the management API answers, besides this machine; repeatable (default: this machine only)")
|
||||||
|
c.Flags().StringArrayVar(&clients, "api-client", nil,
|
||||||
|
"with mutual TLS, the common name of a certificate the management API answers; repeatable (default: any from --tls-ca)")
|
||||||
|
c.Flags().BoolVar(&allowPrivate, "allow-private-sources", false,
|
||||||
|
"let downloads reach loopback, private and link-local addresses")
|
||||||
c.Flags().StringVar(&tlsc.CA, "tls-ca", "", "CA bundle that management clients must present a certificate from")
|
c.Flags().StringVar(&tlsc.CA, "tls-ca", "", "CA bundle that management clients must present a certificate from")
|
||||||
c.Flags().StringVar(&tlsc.Cert, "tls-cert", "", "this mirror's certificate")
|
c.Flags().StringVar(&tlsc.Cert, "tls-cert", "", "this mirror's certificate")
|
||||||
c.Flags().StringVar(&tlsc.Key, "tls-key", "", "this mirror's private key")
|
c.Flags().StringVar(&tlsc.Key, "tls-key", "", "this mirror's private key")
|
||||||
@@ -435,14 +453,38 @@ func parseAllow(list []string) ([]netip.Prefix, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// onlyFrom answers only this machine and the addresses allowed, when any
|
// apiGate decides who reaches the management API. With mutual TLS the
|
||||||
// are: the management API can delete models and start downloads, and a
|
// certificate decides, narrowed by --api-allow when given. Without it, only
|
||||||
// mirror on its own machine is reached over the network by one control plane
|
// this machine and the addresses allowed: nothing allowed used to mean anyone,
|
||||||
// and no one else. Nothing allowed is the old behaviour, anyone.
|
// and every node that could reach the port could delete models.
|
||||||
func onlyFrom(allowed []netip.Prefix, next http.Handler) http.Handler {
|
func apiGate(mtls bool, allowed []netip.Prefix, next http.Handler) http.Handler {
|
||||||
if len(allowed) == 0 {
|
if mtls && len(allowed) == 0 {
|
||||||
return next
|
return next
|
||||||
}
|
}
|
||||||
|
return onlyFrom(allowed, next)
|
||||||
|
}
|
||||||
|
|
||||||
|
// onlyClients answers only certificates with one of these common names, when
|
||||||
|
// any are given. The handshake has already checked the certificate against
|
||||||
|
// the root; this narrows which of the root's certificates may manage.
|
||||||
|
func onlyClients(names []string, next http.Handler) http.Handler {
|
||||||
|
if len(names) == 0 {
|
||||||
|
return next
|
||||||
|
}
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 ||
|
||||||
|
!slices.Contains(names, r.TLS.VerifiedChains[0][0].Subject.CommonName) {
|
||||||
|
http.Error(w, "this mirror is managed by its control plane only", http.StatusForbidden)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
next.ServeHTTP(w, r)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// onlyFrom answers only this machine and the addresses allowed: the management
|
||||||
|
// API can delete models and start downloads, and a mirror on its own machine
|
||||||
|
// is reached over the network by one control plane and no one else.
|
||||||
|
func onlyFrom(allowed []netip.Prefix, next http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||||
a, perr := netip.ParseAddr(host)
|
a, perr := netip.ParseAddr(host)
|
||||||
|
|||||||
@@ -0,0 +1,66 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/tls"
|
||||||
|
"crypto/x509"
|
||||||
|
"crypto/x509/pkix"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/netip"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTheMirrorAPIAnswersOnlyThisMachineUnlessTold(t *testing.T) {
|
||||||
|
ok := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {})
|
||||||
|
call := func(h http.Handler, from string) int {
|
||||||
|
r := httptest.NewRequest("DELETE", "/v1/models/m/v", nil)
|
||||||
|
r.RemoteAddr = from + ":5000"
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, r)
|
||||||
|
return w.Code
|
||||||
|
}
|
||||||
|
cp := []netip.Prefix{netip.MustParsePrefix("10.0.0.140/32")}
|
||||||
|
|
||||||
|
// Nothing allowed and no TLS used to mean anyone: every node could delete
|
||||||
|
// models.
|
||||||
|
if got := call(apiGate(false, nil, ok), "10.0.0.141"); got != http.StatusForbidden {
|
||||||
|
t.Errorf("a node reached an unconfigured mirror API: %d", got)
|
||||||
|
}
|
||||||
|
if got := call(apiGate(false, nil, ok), "127.0.0.1"); got != http.StatusOK {
|
||||||
|
t.Errorf("the mirror's own machine was refused: %d", got)
|
||||||
|
}
|
||||||
|
if got := call(apiGate(false, cp, ok), "10.0.0.140"); got != http.StatusOK {
|
||||||
|
t.Errorf("the allowed control plane was refused: %d", got)
|
||||||
|
}
|
||||||
|
if got := call(apiGate(false, cp, ok), "10.0.0.141"); got != http.StatusForbidden {
|
||||||
|
t.Errorf("a node outside the allowlist got through: %d", got)
|
||||||
|
}
|
||||||
|
// With mutual TLS the certificate decides.
|
||||||
|
if got := call(apiGate(true, nil, ok), "10.0.0.141"); got != http.StatusOK {
|
||||||
|
t.Errorf("mTLS without an allowlist refused a client: %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOnlyANamedCertificateManagesTheMirror(t *testing.T) {
|
||||||
|
ok := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {})
|
||||||
|
with := func(cn string) *http.Request {
|
||||||
|
r := httptest.NewRequest("GET", "/", nil)
|
||||||
|
if cn != "" {
|
||||||
|
r.TLS = &tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{{{Subject: pkix.Name{CommonName: cn}}}}}
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
h := onlyClients([]string{"control-plane"}, ok)
|
||||||
|
for cn, want := range map[string]int{"control-plane": 200, "node01": 403, "": 403} {
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, with(cn))
|
||||||
|
if w.Code != want {
|
||||||
|
t.Errorf("%q: %d, expected %d", cn, w.Code, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
onlyClients(nil, ok).ServeHTTP(w, with("node01"))
|
||||||
|
if w.Code != 200 {
|
||||||
|
t.Error("no names given narrowed it anyway")
|
||||||
|
}
|
||||||
|
}
|
||||||
+149
-2
@@ -3,7 +3,11 @@ package cli
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
"os/user"
|
"os/user"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/knadh/koanf/parsers/yaml"
|
"github.com/knadh/koanf/parsers/yaml"
|
||||||
@@ -23,12 +27,15 @@ import (
|
|||||||
// minutes of outage. For a fixed production set that is a reasonable trade; for
|
// minutes of outage. For a fixed production set that is a reasonable trade; for
|
||||||
// a platform whose promise is that people can try things, it is the difference
|
// a platform whose promise is that people can try things, it is the difference
|
||||||
// between trying one and not bothering.
|
// between trying one and not bothering.
|
||||||
func newModelCmd(client func() *ctlClient) *cobra.Command {
|
func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||||
c := &cobra.Command{
|
c := &cobra.Command{
|
||||||
Use: "model",
|
Use: "model",
|
||||||
Short: "Add or remove models while the cluster is running",
|
Short: "Add or remove models while the cluster is running",
|
||||||
}
|
}
|
||||||
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client))
|
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||||
|
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
|
||||||
|
newTargetCmd(client, asJSON), newAdaptersCmd(client),
|
||||||
|
newModelServeCmd(client))
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -126,6 +133,10 @@ func newModelRemoveCmd(client func() *ctlClient) *cobra.Command {
|
|||||||
|
|
||||||
// actor names who made a change, for the audit trail on a stored model.
|
// actor names who made a change, for the audit trail on a stored model.
|
||||||
func actor() string {
|
func actor() string {
|
||||||
|
// Under sudo, the person who ran it rather than root.
|
||||||
|
if u := os.Getenv("SUDO_USER"); u != "" && os.Geteuid() == 0 {
|
||||||
|
return u
|
||||||
|
}
|
||||||
if u, err := user.Current(); err == nil && u.Username != "" {
|
if u, err := user.Current(); err == nil && u.Username != "" {
|
||||||
return u.Username
|
return u.Username
|
||||||
}
|
}
|
||||||
@@ -186,3 +197,139 @@ func newScaleCmd(client func() *ctlClient) *cobra.Command {
|
|||||||
"how long an on-demand model stays up unasked-for; zero takes the cluster default")
|
"how long an on-demand model stays up unasked-for; zero takes the cluster default")
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newResourcesCmd(client func() *ctlClient) *cobra.Command {
|
||||||
|
var memory, cores int
|
||||||
|
var profile string
|
||||||
|
c := &cobra.Command{
|
||||||
|
Use: "resources <model> --memory GB [--cores N]",
|
||||||
|
Short: "Set what a model reserves of a node: memory, and CPU cores",
|
||||||
|
Long: "--memory is the host RAM a replica needs: for a model with no GPU, its\n" +
|
||||||
|
"weights, KV cache and engine together. Placement reserves it on the node,\n" +
|
||||||
|
"so a node holds only as many replicas as it has memory for.\n\n" +
|
||||||
|
"--cores is the CPU cores a replica with no GPU computes on. They are\n" +
|
||||||
|
"reserved the same way, and become the engine's thread count, so two\n" +
|
||||||
|
"replicas on one node stop competing for the same cores. Zero reserves\n" +
|
||||||
|
"none and leaves the thread count to the node.\n\n" +
|
||||||
|
"Running replicas keep what they were started with; `llmgw ctl restart\n" +
|
||||||
|
"--model M` moves them to the new figures.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
if !cmd.Flags().Changed("memory") {
|
||||||
|
return fmt.Errorf("--memory is required")
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(adminapi.ResourcesRequest{Profile: profile, MemoryGB: memory, CPUCores: cores})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var m catalog.Model
|
||||||
|
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/resources", body, &m); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, p := range m.Profiles {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "%s/%s: %d GB memory, %d cores\n", m.Name, p.Name, p.MinMemoryGB, p.CPUCores)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "Running replicas keep their old figures until restarted: llmgw ctl restart --model %s\n", m.Name)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c.Flags().IntVar(&memory, "memory", 0, "host RAM a replica needs, in GB")
|
||||||
|
c.Flags().IntVar(&cores, "cores", 0, "CPU cores a replica with no GPU computes on; 0 reserves none")
|
||||||
|
c.Flags().StringVar(&profile, "profile", "", "one profile; every profile when left out")
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func newClassCmd(client func() *ctlClient) *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "class <model> critical|standard|opportunistic|default",
|
||||||
|
Short: "Set how much a model's capacity is worth against another's",
|
||||||
|
Long: "When a model cannot be placed, replicas of a lower class are stopped to\n" +
|
||||||
|
"make room for it:\n\n" +
|
||||||
|
" critical never gives up the replicas of its minimum\n" +
|
||||||
|
" standard gives way to critical\n" +
|
||||||
|
" opportunistic gives way to both; the default for an on-demand model\n\n" +
|
||||||
|
"Within a class, a model below its minimum may take what another grew into\n" +
|
||||||
|
"under load, but not its minimum. \"default\" goes back to what the model's\n" +
|
||||||
|
"scaling implies: opportunistic when on demand, standard otherwise.",
|
||||||
|
Args: cobra.ExactArgs(2),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
class := args[1]
|
||||||
|
if class == "default" {
|
||||||
|
class = ""
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(map[string]string{"class": class})
|
||||||
|
var res map[string]string
|
||||||
|
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/class", body, &res); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "%s is %s\n", res["model"], res["class"])
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newAdaptersCmd(client func() *ctlClient) *cobra.Command {
|
||||||
|
var add, remove []string
|
||||||
|
c := &cobra.Command{
|
||||||
|
Use: "adapters <model> [--add NAME=URL[@SCALE]] [--remove NAME]",
|
||||||
|
Short: "List, add or remove a model's LoRA adapters",
|
||||||
|
Long: "An adapter is asked for by its NAME as though it were a model, and is served\n" +
|
||||||
|
"by this model's replicas with it applied. URL is the adapter file, usually in\n" +
|
||||||
|
"the mirror; it must be a GGUF LoRA adapter for this model's architecture,\n" +
|
||||||
|
"which is checked before it is attached. Replicas load a changed set when\n" +
|
||||||
|
"restarted: llmgw ctl restart --model MODEL.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
name := args[0]
|
||||||
|
var models []adminapi.ModelView
|
||||||
|
if err := client().get(cmd.Context(), "/api/v1/models", &models); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var current []catalog.Adapter
|
||||||
|
found := false
|
||||||
|
for _, m := range models {
|
||||||
|
if m.Name == name {
|
||||||
|
current, found = m.Adapters, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return fmt.Errorf("no model %q", name)
|
||||||
|
}
|
||||||
|
out := cmd.OutOrStdout()
|
||||||
|
if len(add) == 0 && len(remove) == 0 {
|
||||||
|
if len(current) == 0 {
|
||||||
|
fmt.Fprintf(out, "%s has no adapters\n", name)
|
||||||
|
}
|
||||||
|
for _, a := range current {
|
||||||
|
fmt.Fprintf(out, "%s\tscale %g\t%s\n", a.Name, a.EffectiveScale(), a.Source)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
next := slices.DeleteFunc(slices.Clone(current), func(a catalog.Adapter) bool { return slices.Contains(remove, a.Name) })
|
||||||
|
for _, spec := range add {
|
||||||
|
n, rest, ok := strings.Cut(spec, "=")
|
||||||
|
if !ok || n == "" || rest == "" {
|
||||||
|
return fmt.Errorf("--add %q is not NAME=URL[@SCALE]", spec)
|
||||||
|
}
|
||||||
|
ad := catalog.Adapter{Name: n, Source: rest}
|
||||||
|
if i := strings.LastIndex(rest, "@"); i > 0 {
|
||||||
|
if s, err := strconv.ParseFloat(rest[i+1:], 64); err == nil {
|
||||||
|
ad.Source, ad.Scale = rest[:i], s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
next = append(next, ad)
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(adminapi.AdaptersRequest{Adapters: next})
|
||||||
|
var res adminapi.AdaptersRequest
|
||||||
|
if err := client().post(cmd.Context(), "/api/v1/models/"+name+"/adapters", body, &res); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "%s serves %d adapter(s); restart its replicas to load them: llmgw ctl restart --model %s\n",
|
||||||
|
name, len(res.Adapters), name)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c.Flags().StringArrayVar(&add, "add", nil, "NAME=URL[@SCALE]; repeatable")
|
||||||
|
c.Flags().StringArrayVar(&remove, "remove", nil, "an adapter's name; repeatable")
|
||||||
|
return c
|
||||||
|
}
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||||
|
var (
|
||||||
|
req modelserve.Request
|
||||||
|
onDemand bool
|
||||||
|
)
|
||||||
|
c := &cobra.Command{
|
||||||
|
Use: "serve OWNER/REPO",
|
||||||
|
Short: "Take a model from the registry to a replica that answers, in one command",
|
||||||
|
Long: "Picks the GGUF file (--quant, Q4_K_M by default, or --file), has the mirror\n" +
|
||||||
|
"fetch it and checks what arrived against the registry's checksum, checks\n" +
|
||||||
|
"that the nodes' llama.cpp can load it and sizes its memory from its own\n" +
|
||||||
|
"header, adds it to the catalog, waits for a replica, and asks it to say\n" +
|
||||||
|
"hello. Each step is shown as it happens; the first that fails stops it.\n\n" +
|
||||||
|
" llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" +
|
||||||
|
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" +
|
||||||
|
" llmgw ctl model serve owner/big-GGUF --on-demand",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
req.Repo = args[0]
|
||||||
|
if onDemand {
|
||||||
|
req.Replicas = 0
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(req)
|
||||||
|
var job modelserve.Job
|
||||||
|
if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out := cmd.OutOrStdout()
|
||||||
|
fmt.Fprintf(out, "job %s\n", job.ID)
|
||||||
|
shown := map[string]string{}
|
||||||
|
for {
|
||||||
|
for _, s := range job.Steps {
|
||||||
|
line := s.State + " " + s.Detail
|
||||||
|
if shown[s.Name] == line {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
shown[s.Name] = line
|
||||||
|
mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok",
|
||||||
|
modelserve.Failed: "!!"}[s.State]
|
||||||
|
fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail)
|
||||||
|
}
|
||||||
|
switch job.State {
|
||||||
|
case modelserve.Done:
|
||||||
|
fmt.Fprintf(out, "\n%s is served.\n", job.Model)
|
||||||
|
return nil
|
||||||
|
case modelserve.Failed:
|
||||||
|
return errors.New(job.Error)
|
||||||
|
}
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
f := c.Flags()
|
||||||
|
f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)")
|
||||||
|
f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant")
|
||||||
|
f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
|
||||||
|
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
|
||||||
|
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
|
||||||
|
f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run")
|
||||||
|
f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for")
|
||||||
|
f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots")
|
||||||
|
f.IntVar(&req.CPUCores, "cores", 0, "CPU cores a replica computes on; 0 takes half the smallest node it fits")
|
||||||
|
f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use")
|
||||||
|
return c
|
||||||
|
}
|
||||||
@@ -0,0 +1,234 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newQuotaCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||||
|
c := &cobra.Command{
|
||||||
|
Use: "quota",
|
||||||
|
Short: "What parts of the tenancy tree are guaranteed of the cluster, and may hold",
|
||||||
|
Long: "A capacity quota belongs to an organization, team or project, and counts the\n" +
|
||||||
|
"GPUs, CPU cores and memory held by replicas of the models it owns (see\n" +
|
||||||
|
"`llmgw ctl model owner`), and of those its teams and projects own.\n\n" +
|
||||||
|
"The guarantee is kept for it: when it holds less and a model of its needs\n" +
|
||||||
|
"room, replicas other scopes hold beyond their own guarantees are stopped to\n" +
|
||||||
|
"make it — whatever their class. What it holds within its guarantee is\n" +
|
||||||
|
"never taken by another scope's model. Beyond it, it borrows: free room is\n" +
|
||||||
|
"used, and given back when asked for.\n\n" +
|
||||||
|
"The limit is the most it may hold, borrowing included; a model that would\n" +
|
||||||
|
"take it past is not given another replica.\n\n" +
|
||||||
|
"Scopes are written org:Acme, team:Acme/ml or project:Acme/ml/training, or\n" +
|
||||||
|
"with IDs. Changes take effect within half a minute.",
|
||||||
|
}
|
||||||
|
c.AddCommand(newQuotaListCmd(client, asJSON), newQuotaSetCmd(client), newQuotaClearCmd(client))
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func newQuotaListCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "list",
|
||||||
|
Short: "Every quota, with what its scope holds now",
|
||||||
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
var qs []adminapi.QuotaView
|
||||||
|
if err := client().get(cmd.Context(), "/api/v1/quotas", &qs); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
return json.NewEncoder(cmd.OutOrStdout()).Encode(qs)
|
||||||
|
}
|
||||||
|
if len(qs) == 0 {
|
||||||
|
fmt.Fprintln(cmd.OutOrStdout(), "No capacity quotas: everything shares the cluster by class alone.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
w := tab()
|
||||||
|
fmt.Fprintln(w, "SCOPE\tNAME\tGPUS\tCORES\tMEMORY GB")
|
||||||
|
for _, q := range qs {
|
||||||
|
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", q.Scope, q.Name,
|
||||||
|
quotaCell(q.Holds.GPUs, q.Guarantee.GPUs, q.Limit.GPUs),
|
||||||
|
quotaCell(q.Holds.CPUCores, q.Guarantee.CPUCores, q.Limit.CPUCores),
|
||||||
|
quotaCell(q.Holds.MemoryGB, q.Guarantee.MemoryGB, q.Limit.MemoryGB))
|
||||||
|
}
|
||||||
|
if err := w.Flush(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintln(cmd.OutOrStdout(), "\nEach cell: holds / guaranteed (at most limit). - is not governed.")
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// quotaCell is "6 / 8 (≤ 16)": what is held, what is guaranteed, the limit.
|
||||||
|
func quotaCell(holds int, guarantee, limit *int) string {
|
||||||
|
if guarantee == nil && limit == nil {
|
||||||
|
return "-"
|
||||||
|
}
|
||||||
|
out := strconv.Itoa(holds) + " / "
|
||||||
|
if guarantee != nil {
|
||||||
|
out += strconv.Itoa(*guarantee)
|
||||||
|
} else {
|
||||||
|
out += "-"
|
||||||
|
}
|
||||||
|
if limit != nil {
|
||||||
|
out += fmt.Sprintf(" (≤ %d)", *limit)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func newQuotaSetCmd(client func() *ctlClient) *cobra.Command {
|
||||||
|
var gpus, cores, mem, lgpus, lcores, lmem int
|
||||||
|
c := &cobra.Command{
|
||||||
|
Use: "set SCOPE [--gpus N] [--cores N] [--memory GB] [--limit-gpus N] ...",
|
||||||
|
Short: "Set a scope's guarantee and limit, replacing what it had",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
f := cmd.Flags()
|
||||||
|
pick := func(name string, v int) *int {
|
||||||
|
if !f.Changed(name) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &v
|
||||||
|
}
|
||||||
|
req := adminapi.QuotaRequest{Scope: args[0],
|
||||||
|
Guarantee: store.Resources{GPUs: pick("gpus", gpus), CPUCores: pick("cores", cores), MemoryGB: pick("memory", mem)},
|
||||||
|
Limit: store.Resources{GPUs: pick("limit-gpus", lgpus), CPUCores: pick("limit-cores", lcores), MemoryGB: pick("limit-memory", lmem)},
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(req)
|
||||||
|
var res map[string]string
|
||||||
|
if err := client().put(cmd.Context(), "/api/v1/quotas", body, &res); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "%s: set; it takes effect within half a minute\n", res["scope"])
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
f := c.Flags()
|
||||||
|
f.IntVar(&gpus, "gpus", 0, "GPUs guaranteed")
|
||||||
|
f.IntVar(&cores, "cores", 0, "CPU cores guaranteed")
|
||||||
|
f.IntVar(&mem, "memory", 0, "memory guaranteed, in GB")
|
||||||
|
f.IntVar(&lgpus, "limit-gpus", 0, "the most GPUs it may hold")
|
||||||
|
f.IntVar(&lcores, "limit-cores", 0, "the most CPU cores it may hold")
|
||||||
|
f.IntVar(&lmem, "limit-memory", 0, "the most memory it may hold, in GB")
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func newQuotaClearCmd(client func() *ctlClient) *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "clear SCOPE",
|
||||||
|
Short: "Remove a scope's quota",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
var res map[string]string
|
||||||
|
if err := client().delete(cmd.Context(), "/api/v1/quotas/"+url.PathEscape(args[0]), &res); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "%s: quota removed\n", res["removed"])
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newOwnerCmd(client func() *ctlClient) *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "owner <model> SCOPE|shared",
|
||||||
|
Short: "Set whose capacity quota a model counts against",
|
||||||
|
Long: "SCOPE is org:Acme, team:Acme/ml or project:Acme/ml/training. A shared\n" +
|
||||||
|
"model belongs to the cluster and counts against nobody's quota.",
|
||||||
|
Args: cobra.ExactArgs(2),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
owner := args[1]
|
||||||
|
if owner == "shared" {
|
||||||
|
owner = ""
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(map[string]string{"owner": owner})
|
||||||
|
var res map[string]string
|
||||||
|
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/owner", body, &res); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if res["owner"] == "" {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "%s is shared\n", res["model"])
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "%s counts against %s\n", res["model"], res["owner"])
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTargetCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||||
|
var tps float64
|
||||||
|
var apply bool
|
||||||
|
c := &cobra.Command{
|
||||||
|
Use: "target <model> --tokens-per-sec N [--apply]",
|
||||||
|
Short: "How many replicas a model needs for a throughput, and whether they fit",
|
||||||
|
Long: "Sized from what one replica was measured to serve while meeting its SLO\n" +
|
||||||
|
"(`llmgw ctl bench MODEL --save`), on the profile its replicas run on.\n" +
|
||||||
|
"--apply makes that the number it always runs, if the cluster has room.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
if tps <= 0 {
|
||||||
|
return fmt.Errorf("--tokens-per-sec is required")
|
||||||
|
}
|
||||||
|
var v adminapi.TargetView
|
||||||
|
path := "/api/v1/models/" + url.PathEscape(args[0]) + "/target?tokens_per_sec=" +
|
||||||
|
strconv.FormatFloat(tps, 'f', -1, 64)
|
||||||
|
if err := client().get(cmd.Context(), path, &v); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
|
||||||
|
}
|
||||||
|
out := cmd.OutOrStdout()
|
||||||
|
fmt.Fprintf(out, "%s: %.0f tokens/s needs %d replicas at %.1f each (measured on %s); %d running\n",
|
||||||
|
v.Model, v.TokPerSec, v.Needed, v.PerReplica, v.Profile, v.Current)
|
||||||
|
switch {
|
||||||
|
case v.More == 0:
|
||||||
|
fmt.Fprintln(out, "It already has them.")
|
||||||
|
case v.Fits && len(v.Preempts) > 0:
|
||||||
|
fmt.Fprintf(out, "%d more; there is room once replicas of %s are stopped, which its class may do.\n",
|
||||||
|
v.More, strings.Join(v.Preempts, ", "))
|
||||||
|
case v.Fits:
|
||||||
|
fmt.Fprintf(out, "%d more, and there is room for them.\n", v.More)
|
||||||
|
default:
|
||||||
|
fmt.Fprintf(out, "%d more, and there is room for %d:\n", v.More, v.Room)
|
||||||
|
for _, b := range v.Blocked {
|
||||||
|
fmt.Fprintln(out, " "+b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !apply || v.More == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !v.Fits {
|
||||||
|
return fmt.Errorf("not applied: the cluster has no room for them")
|
||||||
|
}
|
||||||
|
var models []adminapi.ModelView
|
||||||
|
if err := client().get(cmd.Context(), "/api/v1/models", &models); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
top := v.Needed
|
||||||
|
for _, m := range models {
|
||||||
|
if m.Name == v.Model {
|
||||||
|
top = max(top, m.Max)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
body, _ := json.Marshal(adminapi.ScaleRequest{Min: v.Needed, Max: top})
|
||||||
|
if err := client().post(cmd.Context(), "/api/v1/models/"+url.PathEscape(v.Model)+"/scale", body, nil); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "Applied: %s runs %d (up to %d under load).\n", v.Model, v.Needed, top)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c.Flags().Float64Var(&tps, "tokens-per-sec", 0, "the throughput to size for")
|
||||||
|
c.Flags().BoolVar(&apply, "apply", false, "set it as the number of replicas the model always runs")
|
||||||
|
return c
|
||||||
|
}
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newReleaseCmd() *cobra.Command {
|
||||||
|
c := &cobra.Command{
|
||||||
|
Use: "release",
|
||||||
|
Short: "Sign releases, and make the key they are signed with",
|
||||||
|
}
|
||||||
|
c.AddCommand(&cobra.Command{
|
||||||
|
Use: "keygen",
|
||||||
|
Short: "Make a release signing key pair",
|
||||||
|
Long: "Prints a public key and a private key. The private key goes into the CI\n" +
|
||||||
|
"secret RELEASE_SIGNING_KEY and nowhere else; the public key into\n" +
|
||||||
|
"internal/release/trusted.pub, which every build from then on carries\n" +
|
||||||
|
"and checks releases against. Run once, on a machine you trust.",
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
|
pub, priv, err := release.GenerateKey()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "public: %s\nprivate: %s\n", pub, priv)
|
||||||
|
fmt.Fprintln(cmd.ErrOrStderr(), "\nThe private key signs every release: put it in the CI secret "+
|
||||||
|
"RELEASE_SIGNING_KEY, then clear this terminal's scrollback.")
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
c.AddCommand(&cobra.Command{
|
||||||
|
Use: "sign FILE",
|
||||||
|
Short: "Sign a release's checksums, writing FILE.sig",
|
||||||
|
Long: "Reads the private key from RELEASE_SIGNING_KEY, never from an argument,\n" +
|
||||||
|
"so it does not land in a shell history or a process list.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
key := os.Getenv("RELEASE_SIGNING_KEY")
|
||||||
|
if key == "" {
|
||||||
|
return errors.New("RELEASE_SIGNING_KEY is not set")
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(args[0])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sig, err := release.Sign(key, data)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if pub := release.TrustedKey(); pub != "" {
|
||||||
|
// Signed with a key this build would not accept is a release
|
||||||
|
// every upgraded machine refuses: stopped here instead.
|
||||||
|
if err := release.Verify(pub, data, sig); err != nil {
|
||||||
|
return fmt.Errorf("the signing key does not match this build's trusted key: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return os.WriteFile(args[0]+".sig", []byte(sig+"\n"), 0o644) //nolint:gosec // public
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return c
|
||||||
|
}
|
||||||
@@ -47,6 +47,8 @@ func Root() *cobra.Command {
|
|||||||
newSimEngineCmd(),
|
newSimEngineCmd(),
|
||||||
newDoctorCmd(),
|
newDoctorCmd(),
|
||||||
newNodeCmd(),
|
newNodeCmd(),
|
||||||
|
newSelfUpgradeCmd(),
|
||||||
|
newReleaseCmd(),
|
||||||
)
|
)
|
||||||
return root
|
return root
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||||
|
)
|
||||||
|
|
||||||
|
// newSelfUpgradeCmd upgrades this machine's llmgw on its own, for the machines
|
||||||
|
// `llmgw ctl upgrade` does not reach: a mirror host, above all, whose service
|
||||||
|
// runs unprivileged with the filesystem read-only and so cannot replace its own
|
||||||
|
// binary the way an agent does.
|
||||||
|
func newSelfUpgradeCmd() *cobra.Command {
|
||||||
|
var releases, service string
|
||||||
|
c := &cobra.Command{
|
||||||
|
Use: "upgrade VERSION [--service UNIT]",
|
||||||
|
Short: "Upgrade this machine's llmgw, and restart the service that runs it",
|
||||||
|
Long: "For a machine the control plane does not upgrade — the model mirror's host.\n" +
|
||||||
|
"The control plane and the nodes are upgraded with `llmgw ctl upgrade`.\n\n" +
|
||||||
|
"Fetches the release for this platform, checks it against the release's\n" +
|
||||||
|
"checksums, keeps the old binary beside the new one, and restarts --service,\n" +
|
||||||
|
"waiting for it to be running again. Run it with sudo.",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
v := args[0]
|
||||||
|
out := cmd.OutOrStdout()
|
||||||
|
if !release.Tag.MatchString(v) {
|
||||||
|
return fmt.Errorf("%q is not a release version (v1.2.3 or v1.2.3-rc.4)", v)
|
||||||
|
}
|
||||||
|
if version.Version == v {
|
||||||
|
fmt.Fprintf(out, "this machine already runs %s\n", v)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if os.Geteuid() != 0 {
|
||||||
|
return errors.New("replacing this machine's binary needs root: run it with sudo")
|
||||||
|
}
|
||||||
|
self, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
backup, err := release.Install(cmd.Context(), nil, releases, v, runtime.GOOS, runtime.GOARCH, self)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "%s -> %s: installed, checksum verified; the old binary is %s\n",
|
||||||
|
version.Version, v, backup)
|
||||||
|
if service == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
//nolint:gosec // the unit name is the operator's own flag
|
||||||
|
if b, err := exec.CommandContext(cmd.Context(), "systemctl", "restart", service).CombinedOutput(); err != nil {
|
||||||
|
return fmt.Errorf("restarting %s: %v: %s", service, err, strings.TrimSpace(string(b)))
|
||||||
|
}
|
||||||
|
// Running, and still running a few seconds on: a unit that starts
|
||||||
|
// and dies on a bad flag is "active" for a moment first.
|
||||||
|
deadline := time.Now().Add(30 * time.Second)
|
||||||
|
steady := 0
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
//nolint:gosec // as above
|
||||||
|
b, _ := exec.CommandContext(cmd.Context(), "systemctl", "is-active", service).Output()
|
||||||
|
if strings.TrimSpace(string(b)) == "active" {
|
||||||
|
if steady++; steady >= 3 {
|
||||||
|
fmt.Fprintf(out, "%s restarted and running\n", service)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
steady = 0
|
||||||
|
}
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%s is not running after the upgrade; see journalctl -u %s, and %s holds the previous binary",
|
||||||
|
service, service, backup)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from")
|
||||||
|
c.Flags().StringVar(&service, "service", "", "the systemd unit to restart afterwards, e.g. llmgw-mirror")
|
||||||
|
return c
|
||||||
|
}
|
||||||
+98
-2
@@ -5,8 +5,10 @@ import (
|
|||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -33,9 +35,11 @@ import (
|
|||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
|
||||||
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
|
||||||
)
|
)
|
||||||
@@ -87,6 +91,21 @@ func newServerCmd() *cobra.Command {
|
|||||||
"or give --endpoint for a router-only run")
|
"or give --endpoint for a router-only run")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Before the router, so its tracer is the configured one.
|
||||||
|
shutdownTracing, err := obs.SetupTracing(cmd.Context(), obs.TracingOptions{
|
||||||
|
Endpoint: cfg.Tracing.Endpoint, SampleRatio: cfg.Tracing.SampleRatio,
|
||||||
|
Service: "llmgw-server", Version: version.Version,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
// Bounded: a collector that is gone must not hold up a shutdown.
|
||||||
|
sctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_ = shutdownTracing(sctx)
|
||||||
|
}()
|
||||||
|
|
||||||
rt, err := router.New(cfg, log)
|
rt, err := router.New(cfg, log)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -131,6 +150,28 @@ func newServerCmd() *cobra.Command {
|
|||||||
// On the same /metrics as the traffic figures: correlating
|
// On the same /metrics as the traffic figures: correlating
|
||||||
// "the cluster was full" with "requests were queueing" should
|
// "the cluster was full" with "requests were queueing" should
|
||||||
// not need two systems.
|
// not need two systems.
|
||||||
|
// When the newest backup was written, so its absence alerts
|
||||||
|
// rather than being found on the day it is needed.
|
||||||
|
backupDir := cfg.BackupDir
|
||||||
|
if backupDir == "" {
|
||||||
|
backupDir = DefaultBackupDir
|
||||||
|
}
|
||||||
|
if err := rt.RegisterMetrics(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||||
|
Name: "llmgw_backup_last_success_timestamp_seconds",
|
||||||
|
Help: "When the newest backup archive was written, as a Unix time; 0 when there is none.",
|
||||||
|
}, func() float64 {
|
||||||
|
if t := NewestBackup(backupDir); !t.IsZero() {
|
||||||
|
return float64(t.Unix())
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
})); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := rt.RegisterMetrics(tlsconf.ExpiryCollector(map[string]string{
|
||||||
|
"agent-server": cfg.AgentTLS.Cert, "agent-ca": cfg.AgentTLS.CA,
|
||||||
|
})); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if err := rt.RegisterMetrics(fleetmetrics.New(ctl)); err != nil {
|
if err := rt.RegisterMetrics(fleetmetrics.New(ctl)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -147,8 +188,17 @@ func newServerCmd() *cobra.Command {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// Kept past this block: serving a model from the registry needs
|
||||||
|
// both the admin API and the mirror, and the mirror is set up
|
||||||
|
// below.
|
||||||
|
var (
|
||||||
|
adminAPI *adminapi.API
|
||||||
|
catalogSvc *modelsvc.Service
|
||||||
|
)
|
||||||
if admin != nil {
|
if admin != nil {
|
||||||
api := adminapi.New(admin)
|
api := adminapi.New(admin)
|
||||||
|
adminAPI = api
|
||||||
|
api.SetReleases(cfg.Releases)
|
||||||
// Catalog changes need somewhere to persist, so this is wired
|
// Catalog changes need somewhere to persist, so this is wired
|
||||||
// only when there is a store. Without one, the endpoints say so
|
// only when there is a store. Without one, the endpoints say so
|
||||||
// rather than accepting a model that vanishes on restart.
|
// rather than accepting a model that vanishes on restart.
|
||||||
@@ -165,6 +215,7 @@ func newServerCmd() *cobra.Command {
|
|||||||
admin.ReplaceCatalog(ctx, built)
|
admin.ReplaceCatalog(ctx, built)
|
||||||
}
|
}
|
||||||
api.SetModels(svc)
|
api.SetModels(svc)
|
||||||
|
catalogSvc = svc
|
||||||
// And the mirror page, so a model that has just been
|
// And the mirror page, so a model that has just been
|
||||||
// fetched can be served without writing YAML by hand.
|
// fetched can be served without writing YAML by hand.
|
||||||
rt.SetCatalogAdmin(svc)
|
rt.SetCatalogAdmin(svc)
|
||||||
@@ -175,6 +226,8 @@ func newServerCmd() *cobra.Command {
|
|||||||
// And what each group of the tenancy tree spent, which is
|
// And what each group of the tenancy tree spent, which is
|
||||||
// the other half of a quota being reviewable.
|
// the other half of a quota being reviewable.
|
||||||
api.SetUsage(st)
|
api.SetUsage(st)
|
||||||
|
// Capacity quotas, set and read over this API.
|
||||||
|
api.SetQuotas(st)
|
||||||
}
|
}
|
||||||
api.Register(adminMux)
|
api.Register(adminMux)
|
||||||
|
|
||||||
@@ -234,7 +287,16 @@ func newServerCmd() *cobra.Command {
|
|||||||
// Browsing the registry is what turns "paste a resolve URL"
|
// Browsing the registry is what turns "paste a resolve URL"
|
||||||
// into "find the model". It needs a way out to the internet
|
// into "find the model". It needs a way out to the internet
|
||||||
// from this machine, which the mirror's own fetching does not.
|
// from this machine, which the mirror's own fetching does not.
|
||||||
rt.SetRegistry(hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout))
|
registry := hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout)
|
||||||
|
rt.SetRegistry(registry)
|
||||||
|
// One command from a registry repository to a replica that
|
||||||
|
// answers: `llmgw ctl model serve`.
|
||||||
|
if adminAPI != nil && catalogSvc != nil {
|
||||||
|
adminAPI.SetServe(modelserve.New(modelserve.Deps{
|
||||||
|
Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin,
|
||||||
|
LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log,
|
||||||
|
}))
|
||||||
|
}
|
||||||
log.Info("model mirror configured",
|
log.Info("model mirror configured",
|
||||||
"api", cfg.Mirror.API, "mtls", cfg.Mirror.TLS.Enabled())
|
"api", cfg.Mirror.API, "mtls", cfg.Mirror.TLS.Enabled())
|
||||||
}
|
}
|
||||||
@@ -246,7 +308,7 @@ func newServerCmd() *cobra.Command {
|
|||||||
// enforces its own per-phase deadlines.
|
// enforces its own per-phase deadlines.
|
||||||
}
|
}
|
||||||
adminSrv := &http.Server{
|
adminSrv := &http.Server{
|
||||||
Handler: adminMux,
|
Handler: router.AdminGuard(adminMux),
|
||||||
ReadHeaderTimeout: 10 * time.Second,
|
ReadHeaderTimeout: 10 * time.Second,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -361,6 +423,7 @@ func newServerCmd() *cobra.Command {
|
|||||||
f.StringArrayVar(&endpoints, "endpoint", nil, "static endpoint: id=URL=model[,model] (repeatable)")
|
f.StringArrayVar(&endpoints, "endpoint", nil, "static endpoint: id=URL=model[,model] (repeatable)")
|
||||||
|
|
||||||
cmd.AddCommand(newServerInitCmd())
|
cmd.AddCommand(newServerInitCmd())
|
||||||
|
cmd.AddCommand(newBackupCmd())
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -385,6 +448,20 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
|
|||||||
ccfg.Reconciler.PortRange = [2]int{cfg.Control.PortRange[0], cfg.Control.PortRange[1]}
|
ccfg.Reconciler.PortRange = [2]int{cfg.Control.PortRange[0], cfg.Control.PortRange[1]}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Each replica's engine demands a key only the router has. The secret it
|
||||||
|
// is derived from lives in the store, so every control plane against it —
|
||||||
|
// and this one after a restart — computes the same key for a replica.
|
||||||
|
if h, ok := sink.(interface{ Store() *storepkg.DB }); ok && h.Store() != nil {
|
||||||
|
secret, err := h.Store().ClusterSecret(ctx, "engine-key")
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("reading the engine key secret: %w", err)
|
||||||
|
}
|
||||||
|
ccfg.Reconciler.EngineKey = control.EngineKeys([]byte(secret))
|
||||||
|
} else {
|
||||||
|
log.Warn("engines start without a key: there is no store to keep the secret " +
|
||||||
|
"they are derived from, so anyone who can reach a node's ports reaches its models")
|
||||||
|
}
|
||||||
|
|
||||||
// The service needs the inventory and the status sink, both of which the
|
// The service needs the inventory and the status sink, both of which the
|
||||||
// controller owns; the controller needs the service as its dispatcher. The
|
// controller owns; the controller needs the service as its dispatcher. The
|
||||||
// cycle is broken by giving the controller a dispatcher that forwards to
|
// cycle is broken by giving the controller a dispatcher that forwards to
|
||||||
@@ -408,6 +485,12 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
|
|||||||
if st := h.Store(); st != nil {
|
if st := h.Store(); st != nil {
|
||||||
lead = control.NewLeadership(st, cfg.Control.LeaseTTL, log)
|
lead = control.NewLeadership(st, cfg.Control.LeaseTTL, log)
|
||||||
ctl.SetLeadership(lead)
|
ctl.SetLeadership(lead)
|
||||||
|
// The tenancy tree's capacity quotas: what each part is
|
||||||
|
// guaranteed and may hold.
|
||||||
|
ctl.Reconciler().SetQuotas(control.QuotaPolicy(st, log))
|
||||||
|
// And what each model was measured to serve, for scaling and for
|
||||||
|
// choosing between profiles.
|
||||||
|
ctl.Reconciler().SetMeasurements(control.Measurements(st, log))
|
||||||
log.Info("competing to be the control plane that places work",
|
log.Info("competing to be the control plane that places work",
|
||||||
"owner", lead.Owner())
|
"owner", lead.Owner())
|
||||||
}
|
}
|
||||||
@@ -507,6 +590,19 @@ func (r *dispatcherRef) FabricTest(ctx context.Context, nodeID string, req *llmg
|
|||||||
return ft.FabricTest(ctx, nodeID, req)
|
return ft.FabricTest(ctx, nodeID, req)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Upgrade forwards a node upgrade, as Logs does.
|
||||||
|
func (r *dispatcherRef) Upgrade(ctx context.Context, nodeID, version, releases string) error {
|
||||||
|
d, err := r.get()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
u, ok := d.(control.Upgrader)
|
||||||
|
if !ok {
|
||||||
|
return errors.New("this dispatcher cannot upgrade nodes")
|
||||||
|
}
|
||||||
|
return u.Upgrade(ctx, nodeID, version, releases)
|
||||||
|
}
|
||||||
|
|
||||||
func (r *dispatcherRef) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
|
func (r *dispatcherRef) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
|
||||||
d, err := r.get()
|
d, err := r.get()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -94,6 +94,14 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) {
|
|||||||
fmt.Fprintf(out, "\nMODELS %d of %d have the replicas they asked for\n",
|
fmt.Fprintf(out, "\nMODELS %d of %d have the replicas they asked for\n",
|
||||||
s.Models.Satisfied, s.Models.Total)
|
s.Models.Satisfied, s.Models.Total)
|
||||||
|
|
||||||
|
if len(s.Parked) > 0 {
|
||||||
|
fmt.Fprintln(out, "\nPARKED — failed every time they started; not retried until fixed")
|
||||||
|
for _, p := range s.Parked {
|
||||||
|
fmt.Fprintf(out, " %s (%d failures): %s\n", p.Model, p.Failures, p.Reason)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out, "\n Once fixed: llmgw ctl retry <model>")
|
||||||
|
}
|
||||||
|
|
||||||
if len(s.Unplaceable) > 0 {
|
if len(s.Unplaceable) > 0 {
|
||||||
fmt.Fprintln(out, "\nNOT PLACED")
|
fmt.Fprintln(out, "\nNOT PLACED")
|
||||||
for _, m := range s.Unplaceable {
|
for _, m := range s.Unplaceable {
|
||||||
|
|||||||
@@ -389,17 +389,22 @@ func (in *serverInit) runInitTUI(ctx context.Context, out io.Writer) error {
|
|||||||
return fmt.Errorf("stopped; run init again to finish what was started")
|
return fmt.Errorf("stopped; run init again to finish what was started")
|
||||||
}
|
}
|
||||||
|
|
||||||
// The whole run is kept, root's only, since it holds the password. On a
|
// The whole run is kept, root's only, with the password taken out: it is
|
||||||
// failure it is printed too, for the lines the screen had no room for;
|
// shown once, on the screen, and a file on disk would outlive that. On a
|
||||||
// otherwise the screen's last frame says it, and only what must outlive
|
// failure the run is printed too, for the lines the screen had no room
|
||||||
// the screen is printed again: the password, and what to do next.
|
// for; otherwise the screen's last frame says it, and only what must
|
||||||
|
// outlive the screen is printed again: the password, and what to do next.
|
||||||
logMu.Lock()
|
logMu.Lock()
|
||||||
defer logMu.Unlock()
|
defer logMu.Unlock()
|
||||||
whole := strings.Join(log, "\n") + "\n"
|
whole := strings.Join(log, "\n") + "\n"
|
||||||
|
kept := whole
|
||||||
|
if fm.password != "" {
|
||||||
|
kept = strings.ReplaceAll(kept, fm.password, "[password shown once, not kept]")
|
||||||
|
}
|
||||||
saved := ""
|
saved := ""
|
||||||
if in.stateDir != "" {
|
if in.stateDir != "" {
|
||||||
p := filepath.Join(in.stateDir, "init.log")
|
p := filepath.Join(in.stateDir, "init.log")
|
||||||
if err := os.WriteFile(p, []byte(whole), 0o600); err == nil {
|
if err := os.WriteFile(p, []byte(kept), 0o600); err == nil {
|
||||||
saved = p
|
saved = p
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -67,6 +67,10 @@ type Server struct {
|
|||||||
|
|
||||||
// Monitoring is where the dashboard reads its history from.
|
// Monitoring is where the dashboard reads its history from.
|
||||||
Monitoring Monitoring `koanf:"monitoring"`
|
Monitoring Monitoring `koanf:"monitoring"`
|
||||||
|
// Tracing sends a trace of each inference request to an OpenTelemetry
|
||||||
|
// collector. Off unless an endpoint is set here or in
|
||||||
|
// OTEL_EXPORTER_OTLP_ENDPOINT.
|
||||||
|
Tracing Tracing `koanf:"tracing"`
|
||||||
|
|
||||||
// Mirror points the control plane at a model mirror it can administer.
|
// Mirror points the control plane at a model mirror it can administer.
|
||||||
// Empty means there is none, and /admin/mirror stays unmounted.
|
// Empty means there is none, and /admin/mirror stays unmounted.
|
||||||
@@ -81,6 +85,16 @@ type Server struct {
|
|||||||
// one leaves that engine to be installed by hand.
|
// one leaves that engine to be installed by hand.
|
||||||
Engines map[string]string `koanf:"engines"`
|
Engines map[string]string `koanf:"engines"`
|
||||||
|
|
||||||
|
// BackupDir is where `llmgw server backup` writes, and where the newest
|
||||||
|
// archive's age is read for llmgw_backup_last_success_timestamp_seconds.
|
||||||
|
// Empty means /var/backups/llmgw-server.
|
||||||
|
BackupDir string `koanf:"backup_dir"`
|
||||||
|
|
||||||
|
// Releases is where `llmgw ctl upgrade` fetches release binaries from,
|
||||||
|
// for this machine and — through their agents — every node. Empty means
|
||||||
|
// the project's own releases.
|
||||||
|
Releases string `koanf:"releases"`
|
||||||
|
|
||||||
Router Router `koanf:"router"`
|
Router Router `koanf:"router"`
|
||||||
Control Control `koanf:"control"`
|
Control Control `koanf:"control"`
|
||||||
Models []catalog.Model `koanf:"models"`
|
Models []catalog.Model `koanf:"models"`
|
||||||
@@ -110,6 +124,21 @@ type Monitoring struct {
|
|||||||
Timeout time.Duration `koanf:"timeout"`
|
Timeout time.Duration `koanf:"timeout"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Tracing is where request traces go, over OTLP/HTTP.
|
||||||
|
//
|
||||||
|
// One span per inference request, with admission, scheduling and each attempt
|
||||||
|
// at an engine beneath it, and the trace carried on to the engine in a
|
||||||
|
// traceparent header. The request's X-Request-Id is on the span, so a trace
|
||||||
|
// and `llmgw ctl request` name the same request.
|
||||||
|
type Tracing struct {
|
||||||
|
// Endpoint is the collector, for example http://otel.internal:4318.
|
||||||
|
// Empty falls back to OTEL_EXPORTER_OTLP_ENDPOINT, and to no tracing.
|
||||||
|
Endpoint string `koanf:"endpoint"`
|
||||||
|
// SampleRatio is the share of requests traced, 0 to 1, when the caller
|
||||||
|
// did not decide it with a sampled traceparent of its own. Zero takes 1.
|
||||||
|
SampleRatio float64 `koanf:"sample_ratio"`
|
||||||
|
}
|
||||||
|
|
||||||
// Mirror is a model mirror this control plane administers over its API.
|
// Mirror is a model mirror this control plane administers over its API.
|
||||||
//
|
//
|
||||||
// The mirror is a separate machine with a large disk, not part of the gateway,
|
// The mirror is a separate machine with a large disk, not part of the gateway,
|
||||||
@@ -349,7 +378,12 @@ type Admission struct {
|
|||||||
|
|
||||||
QueueTTL time.Duration `koanf:"queue_ttl"`
|
QueueTTL time.Duration `koanf:"queue_ttl"`
|
||||||
QueueCapacityPerBand int `koanf:"queue_capacity_per_band"`
|
QueueCapacityPerBand int `koanf:"queue_capacity_per_band"`
|
||||||
Interval time.Duration `koanf:"interval"`
|
// QueueCapacityTotal bounds everything queued across bands. Reached, a
|
||||||
|
// request is not refused while a lower band has any queued: the newest of
|
||||||
|
// the lowest band is turned away instead, with a 503 and Retry-After.
|
||||||
|
// Zero takes half of all the bands' capacities together.
|
||||||
|
QueueCapacityTotal int `koanf:"queue_capacity_total"`
|
||||||
|
Interval time.Duration `koanf:"interval"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prefix configures approximate prefix routing.
|
// Prefix configures approximate prefix routing.
|
||||||
@@ -404,8 +438,11 @@ type Scoring struct {
|
|||||||
|
|
||||||
// Endpoint is one engine replica. Static in phase 1.
|
// Endpoint is one engine replica. Static in phase 1.
|
||||||
type Endpoint struct {
|
type Endpoint struct {
|
||||||
ID string `koanf:"id"`
|
ID string `koanf:"id"`
|
||||||
URL string `koanf:"url"`
|
URL string `koanf:"url"`
|
||||||
|
// APIKey is sent to the engine as a bearer token, for an engine started
|
||||||
|
// with one.
|
||||||
|
APIKey string `koanf:"api_key"`
|
||||||
Models []string `koanf:"models"`
|
Models []string `koanf:"models"`
|
||||||
// Engine names the inference engine, which decides what its metrics are
|
// Engine names the inference engine, which decides what its metrics are
|
||||||
// called. Empty means vLLM's names.
|
// called. Empty means vLLM's names.
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
package adminapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAClaimedActorIsOneShortName(t *testing.T) {
|
||||||
|
for claimed, want := range map[string]string{
|
||||||
|
"ilker": "ilker",
|
||||||
|
"": "unknown",
|
||||||
|
"ilker\nlevel=ERROR msg=x": "ilkerlevelERRORmsgx",
|
||||||
|
strings.Repeat("a", 500): strings.Repeat("a", 64),
|
||||||
|
"ops@example.org": "ops@example.org",
|
||||||
|
"\x1b[31m": "31m",
|
||||||
|
} {
|
||||||
|
r := httptest.NewRequest("POST", "/", nil)
|
||||||
|
r.Header.Set("X-Llmgw-Actor", claimed)
|
||||||
|
if got := actorOf(r); got != want {
|
||||||
|
t.Errorf("%q: %q, expected %q", claimed, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAReservationThatPlacementCannotTrustIsSaid(t *testing.T) {
|
||||||
|
cpu := func(mem int) catalog.Model {
|
||||||
|
return catalog.Model{Name: "m", Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: mem}}}
|
||||||
|
}
|
||||||
|
if w := reservationWarnings(cpu(0), 0); len(w) != 1 || !strings.Contains(w[0], "declares no memory") {
|
||||||
|
t.Errorf("no memory declared: %v", w)
|
||||||
|
}
|
||||||
|
if w := reservationWarnings(cpu(6), 8.2); len(w) != 1 || !strings.Contains(w[0], "uses 8.2 GB and reserves 6") {
|
||||||
|
t.Errorf("under-declared: %v", w)
|
||||||
|
}
|
||||||
|
if w := reservationWarnings(cpu(6), 6.4); len(w) != 0 {
|
||||||
|
t.Errorf("within a tenth is mapped weights, not a wrong figure: %v", w)
|
||||||
|
}
|
||||||
|
gpu := catalog.Model{Name: "g", Profiles: []catalog.Profile{{Name: "tp1", Engine: "vllm", GPUClass: "h100-80g"}}}
|
||||||
|
if w := reservationWarnings(gpu, 3); len(w) != 0 {
|
||||||
|
t.Errorf("a GPU model needs no host memory figure: %v", w)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package adminapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/binary"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ggufStart is the start of a GGUF file with these string keys.
|
||||||
|
func ggufStart(kv ...string) []byte {
|
||||||
|
var b bytes.Buffer
|
||||||
|
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
||||||
|
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
||||||
|
b.WriteString("GGUF")
|
||||||
|
w(uint32(3))
|
||||||
|
w(uint64(0))
|
||||||
|
w(uint64(len(kv) / 2))
|
||||||
|
for i := 0; i < len(kv); i += 2 {
|
||||||
|
str(kv[i])
|
||||||
|
w(uint32(8)) // string
|
||||||
|
str(kv[i+1])
|
||||||
|
}
|
||||||
|
b.Write(make([]byte, 4096))
|
||||||
|
return b.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOnlyAnAdapterForTheModelsArchitectureIsAttached(t *testing.T) {
|
||||||
|
files := map[string][]byte{
|
||||||
|
"/base.gguf": ggufStart("general.architecture", "qwen2"),
|
||||||
|
"/model.gguf": ggufStart("general.architecture", "qwen2", "general.type", "model"),
|
||||||
|
"/llama.gguf": ggufStart("general.architecture", "llama", "general.type", "adapter"),
|
||||||
|
"/lora.gguf": ggufStart("general.architecture", "qwen2", "general.type", "adapter"),
|
||||||
|
}
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusPartialContent)
|
||||||
|
_, _ = w.Write(files[r.URL.Path])
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
models := []catalog.Model{{
|
||||||
|
Name: "coder", Source: srv.URL + "/base.gguf",
|
||||||
|
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||||
|
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 8}},
|
||||||
|
}}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
file, want string
|
||||||
|
code int
|
||||||
|
}{
|
||||||
|
{"/model.gguf", "not a LoRA adapter", http.StatusBadRequest},
|
||||||
|
{"/llama.gguf", "for llama models", http.StatusBadRequest},
|
||||||
|
{"/lora.gguf", "", http.StatusOK},
|
||||||
|
} {
|
||||||
|
a, _ := setup(t, models)
|
||||||
|
fm := &fakeModels{}
|
||||||
|
a.SetModels(fm)
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
a.Register(mux)
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/api/v1/models/coder/adapters",
|
||||||
|
strings.NewReader(`{"adapters":[{"name":"fix","source":"`+srv.URL+tc.file+`"}]}`))
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
mux.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != tc.code || !strings.Contains(rec.Body.String(), tc.want) {
|
||||||
|
t.Errorf("%s: %d %s; want %d mentioning %q", tc.file, rec.Code, rec.Body.String(), tc.code, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,9 +12,16 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||||
|
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
|
||||||
@@ -30,9 +37,15 @@ import (
|
|||||||
|
|
||||||
// API serves the management endpoints.
|
// API serves the management endpoints.
|
||||||
type API struct {
|
type API struct {
|
||||||
|
// releases is where release binaries are fetched from, for upgrades.
|
||||||
|
releases string
|
||||||
|
// serve runs registry-to-replica jobs; nil without a mirror and a store.
|
||||||
|
serve *modelserve.Server
|
||||||
|
|
||||||
models Models
|
models Models
|
||||||
capacity Capacity
|
capacity Capacity
|
||||||
usage Usage
|
usage Usage
|
||||||
|
quotas Quotas
|
||||||
ctl *control.Controller
|
ctl *control.Controller
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -48,6 +61,29 @@ func (a *API) Register(mux *http.ServeMux) {
|
|||||||
mux.HandleFunc("POST /api/v1/models", a.addModel)
|
mux.HandleFunc("POST /api/v1/models", a.addModel)
|
||||||
mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel)
|
mux.HandleFunc("DELETE /api/v1/models/{name}", a.removeModel)
|
||||||
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
|
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
|
||||||
|
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
|
||||||
|
a.registerQuotas(mux)
|
||||||
|
mux.HandleFunc("POST /api/v1/models/{name}/adapters", a.setAdapters)
|
||||||
|
mux.HandleFunc("GET /api/v1/models/{name}/target", a.target)
|
||||||
|
mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if a.models == nil {
|
||||||
|
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var req struct {
|
||||||
|
Class string `json:"class"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m, err := a.models.SetClass(r.Context(), r.PathValue("name"), req.Class, actorOf(r))
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, map[string]string{"model": m.Name, "class": m.EffectiveClass()})
|
||||||
|
})
|
||||||
mux.HandleFunc("PUT /api/v1/models/{name}/capacity", a.putCapacity)
|
mux.HandleFunc("PUT /api/v1/models/{name}/capacity", a.putCapacity)
|
||||||
mux.HandleFunc("GET /api/v1/capacity", a.listCapacity)
|
mux.HandleFunc("GET /api/v1/capacity", a.listCapacity)
|
||||||
mux.HandleFunc("GET /api/v1/usage", a.usageByScope)
|
mux.HandleFunc("GET /api/v1/usage", a.usageByScope)
|
||||||
@@ -74,9 +110,24 @@ func (a *API) Register(mux *http.ServeMux) {
|
|||||||
mux.HandleFunc("POST /api/v1/weights/collect", a.collectWeights)
|
mux.HandleFunc("POST /api/v1/weights/collect", a.collectWeights)
|
||||||
mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement)
|
mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement)
|
||||||
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
|
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
|
||||||
|
mux.HandleFunc("POST /api/v1/nodes/{id}/upgrade", a.upgradeNode)
|
||||||
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
|
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
|
||||||
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
||||||
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
||||||
|
mux.HandleFunc("POST /api/v1/restart", a.restart)
|
||||||
|
mux.HandleFunc("POST /api/v1/models/serve", a.serveModel)
|
||||||
|
mux.HandleFunc("GET /api/v1/jobs/{id}", a.job)
|
||||||
|
mux.HandleFunc("GET /api/v1/requests/{id}", a.request)
|
||||||
|
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
|
||||||
|
mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||||
|
v, ok := a.Headroom(r.PathValue("name"), limit)
|
||||||
|
if !ok {
|
||||||
|
http.Error(w, "no such model", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, v)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- responses ---
|
// --- responses ---
|
||||||
@@ -95,6 +146,8 @@ type NodeView struct {
|
|||||||
// what decides whether a CPU profile can be placed here, so it belongs
|
// what decides whether a CPU profile can be placed here, so it belongs
|
||||||
// where someone looks when a model will not schedule.
|
// where someone looks when a model will not schedule.
|
||||||
MemoryGB int `json:"memory_gb,omitempty"`
|
MemoryGB int `json:"memory_gb,omitempty"`
|
||||||
|
// CPUCount is the machine's CPUs, 0 when the agent did not report them.
|
||||||
|
CPUCount int `json:"cpu_count,omitempty"`
|
||||||
// Engines are the inference engines installed here, as the agent found
|
// Engines are the inference engines installed here, as the agent found
|
||||||
// them. Absent means the agent is older than the probe and said nothing,
|
// them. Absent means the agent is older than the probe and said nothing,
|
||||||
// which is not the same as having none — placement treats it that way too,
|
// which is not the same as having none — placement treats it that way too,
|
||||||
@@ -200,6 +253,11 @@ type LoadView struct {
|
|||||||
// MemPercent is used over used plus available.
|
// MemPercent is used over used plus available.
|
||||||
MemPercent float64 `json:"memory_percent"`
|
MemPercent float64 `json:"memory_percent"`
|
||||||
AgeSec int64 `json:"age_seconds"`
|
AgeSec int64 `json:"age_seconds"`
|
||||||
|
// What the node's sensors say; zero is not measured.
|
||||||
|
CPUTempC float64 `json:"cpu_temp_c,omitempty"`
|
||||||
|
CPUPowerW float64 `json:"cpu_power_w,omitempty"`
|
||||||
|
InletTempC float64 `json:"inlet_temp_c,omitempty"`
|
||||||
|
GPUs []inventory.GPUSensor `json:"gpu_sensors,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// loadFresh is how old a measurement may be and still be shown as current.
|
// loadFresh is how old a measurement may be and still be shown as current.
|
||||||
@@ -213,7 +271,8 @@ func loadView(l *inventory.Load, now time.Time) *LoadView {
|
|||||||
CPUPercent: l.CPUPercent, CPUs: l.CPUCount,
|
CPUPercent: l.CPUPercent, CPUs: l.CPUCount,
|
||||||
Load1: l.Load1, Load5: l.Load5, Load15: l.Load15,
|
Load1: l.Load1, Load5: l.Load5, Load15: l.Load15,
|
||||||
MemUsed: l.MemUsedBytes, MemTotal: l.MemUsedBytes + l.MemAvailBytes,
|
MemUsed: l.MemUsedBytes, MemTotal: l.MemUsedBytes + l.MemAvailBytes,
|
||||||
AgeSec: int64(now.Sub(l.At).Seconds()),
|
AgeSec: int64(now.Sub(l.At).Seconds()),
|
||||||
|
CPUTempC: l.CPUTempC, CPUPowerW: l.CPUPowerW, InletTempC: l.InletTempC, GPUs: l.GPUs,
|
||||||
}
|
}
|
||||||
if v.MemTotal > 0 {
|
if v.MemTotal > 0 {
|
||||||
v.MemPercent = float64(v.MemUsed) * 100 / float64(v.MemTotal)
|
v.MemPercent = float64(v.MemUsed) * 100 / float64(v.MemTotal)
|
||||||
@@ -232,13 +291,52 @@ type ModelView struct {
|
|||||||
Max int `json:"max_replicas"`
|
Max int `json:"max_replicas"`
|
||||||
Ready int `json:"ready_replicas"`
|
Ready int `json:"ready_replicas"`
|
||||||
Total int `json:"total_replicas"`
|
Total int `json:"total_replicas"`
|
||||||
|
// Desired is how many it should have now: Min, or for an on-demand
|
||||||
|
// model one while it is being asked for and none otherwise.
|
||||||
|
Desired int `json:"desired_replicas"`
|
||||||
|
// Failures counts replicas that failed to start in a row.
|
||||||
|
Failures int `json:"failures,omitempty"`
|
||||||
|
// Parked is set once Failures reached the limit; the model is not retried
|
||||||
|
// until its definition changes or `ctl retry` is run. Reason is the last
|
||||||
|
// error.
|
||||||
|
Parked bool `json:"parked,omitempty"`
|
||||||
|
Reason string `json:"reason,omitempty"`
|
||||||
|
// Unplaceable is set when it is short and the last attempt found no node.
|
||||||
|
Unplaceable bool `json:"unplaceable,omitempty"`
|
||||||
Fallbacks []string `json:"fallbacks,omitempty"`
|
Fallbacks []string `json:"fallbacks,omitempty"`
|
||||||
Profiles []string `json:"profiles"`
|
Profiles []string `json:"profiles"`
|
||||||
|
// MemoryGB and CPUCores are what one replica reserves, from its first
|
||||||
|
// profile; MeasuredMemoryGB is the most one of its replicas uses now, 0
|
||||||
|
// when none is measured.
|
||||||
|
MemoryGB int `json:"memory_gb,omitempty"`
|
||||||
|
CPUCores int `json:"cpu_cores,omitempty"`
|
||||||
|
MeasuredMemoryGB float64 `json:"measured_memory_gb,omitempty"`
|
||||||
|
// Warnings are what makes its reservation wrong: nothing declared where
|
||||||
|
// placement needs a figure, or less declared than it uses.
|
||||||
|
Warnings []string `json:"warnings,omitempty"`
|
||||||
|
// Class is its capacity class; Preempted the replicas it lost to a higher
|
||||||
|
// class; HeldFor the model it is giving room to now, if any.
|
||||||
|
Class string `json:"class"`
|
||||||
|
Owner string `json:"owner,omitempty"`
|
||||||
|
// External is where a model served outside the cluster goes, provider
|
||||||
|
// host and its name for the model; empty for one served here.
|
||||||
|
External string `json:"external,omitempty"`
|
||||||
|
// Adapters are the LoRA adapters it serves.
|
||||||
|
Adapters []catalog.Adapter `json:"adapters,omitempty"`
|
||||||
|
Preempted int `json:"preempted,omitempty"`
|
||||||
|
HeldFor string `json:"held_for,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReplicaView is a replica as `ctl replicas` shows it.
|
// ReplicaView is a replica as `ctl replicas` shows it.
|
||||||
type ReplicaView struct {
|
type ReplicaView struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
|
// EngineKey is the key this replica's engine demands, for `ctl bench`
|
||||||
|
// and anyone else holding the admin listener who needs to reach an engine
|
||||||
|
// directly. The client-facing pages never render it.
|
||||||
|
EngineKey string `json:"engine_key,omitempty"`
|
||||||
|
// Restart is "queued" or "replacing" while an operator's restart of this
|
||||||
|
// replica is under way.
|
||||||
|
Restart string `json:"restart,omitempty"`
|
||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
Profile string `json:"profile"`
|
Profile string `json:"profile"`
|
||||||
// Stale is true when the profile has been edited since this replica was
|
// Stale is true when the profile has been edited since this replica was
|
||||||
@@ -303,6 +401,28 @@ type ExplainView struct {
|
|||||||
Rejections []RejectionView `json:"rejections,omitempty"`
|
Rejections []RejectionView `json:"rejections,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// HeadroomView is how many more replicas of a model fit now, and where.
|
||||||
|
type HeadroomView struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
// Current is the replicas holding capacity: starting or ready.
|
||||||
|
Current int `json:"current"`
|
||||||
|
// More is how many more placement would find room for now.
|
||||||
|
More int `json:"more"`
|
||||||
|
// Nodes is where each of those would go.
|
||||||
|
Nodes [][]string `json:"nodes,omitempty"`
|
||||||
|
// Limited is set when More stopped at the limit asked rather than at the
|
||||||
|
// cluster's edge: there may be room for more.
|
||||||
|
Limited bool `json:"limited,omitempty"`
|
||||||
|
// Blocked says why one more does not fit, a line per node considered.
|
||||||
|
Blocked []string `json:"blocked,omitempty"`
|
||||||
|
// TokPerSec is what one more replica was measured to serve; 0 unmeasured.
|
||||||
|
TokPerSec float64 `json:"tok_per_sec,omitempty"`
|
||||||
|
// ByPreempting is how many more would fit by stopping replicas this
|
||||||
|
// model's class may take; Preempts the models those belong to.
|
||||||
|
ByPreempting int `json:"by_preempting,omitempty"`
|
||||||
|
Preempts []string `json:"preempts,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
// RejectionView is one rejected candidate.
|
// RejectionView is one rejected candidate.
|
||||||
type RejectionView struct {
|
type RejectionView struct {
|
||||||
Profile string `json:"profile"`
|
Profile string `json:"profile"`
|
||||||
@@ -337,8 +457,8 @@ func (a *API) Nodes() []NodeView {
|
|||||||
ID: n.ID, Hostname: n.Hostname, MgmtAddr: n.MgmtAddr,
|
ID: n.ID, Hostname: n.Hostname, MgmtAddr: n.MgmtAddr,
|
||||||
State: n.State.String(), Labels: n.Labels,
|
State: n.State.String(), Labels: n.Labels,
|
||||||
GPUTotal: len(n.GPUs), GPUFree: len(inv.FreeGPUs(n.ID)),
|
GPUTotal: len(n.GPUs), GPUFree: len(inv.FreeGPUs(n.ID)),
|
||||||
MemoryGB: n.MemoryTotalMB / 1024,
|
MemoryGB: n.MemoryTotalMB / 1024, CPUCount: n.CPUCount,
|
||||||
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features,
|
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features,
|
||||||
Replicas: replicasOn[n.ID],
|
Replicas: replicasOn[n.ID],
|
||||||
}
|
}
|
||||||
if free, total := n.LocalDisk(); total > 0 {
|
if free, total := n.LocalDisk(); total > 0 {
|
||||||
@@ -434,17 +554,40 @@ func (a *API) Models() []ModelView {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
health := map[string]reconciler.ModelHealth{}
|
||||||
|
for _, h := range a.ctl.Reconciler().ModelHealth() {
|
||||||
|
health[h.Model] = h
|
||||||
|
}
|
||||||
|
// The most one replica of each model uses, summed over its shards.
|
||||||
|
measured := map[string]int64{}
|
||||||
|
for _, rep := range a.Replicas("") {
|
||||||
|
var b int64
|
||||||
|
for _, sh := range rep.Shards {
|
||||||
|
b += sh.MemoryBytes
|
||||||
|
}
|
||||||
|
measured[rep.Model] = max(measured[rep.Model], b)
|
||||||
|
}
|
||||||
out := make([]ModelView, 0)
|
out := make([]ModelView, 0)
|
||||||
for _, m := range a.ctl.Catalog().Models() {
|
for _, m := range a.ctl.Catalog().Models() {
|
||||||
|
h := health[m.Name]
|
||||||
v := ModelView{
|
v := ModelView{
|
||||||
Name: m.Name, Description: m.Description, Aliases: m.Aliases, Source: m.Source,
|
Name: m.Name, Description: m.Description, Aliases: m.Aliases, Source: m.Source,
|
||||||
Min: m.Scaling.Min, Max: m.Scaling.Max,
|
Min: m.Scaling.Min, Max: m.Scaling.Max,
|
||||||
Ready: ready[m.Name], Total: total[m.Name],
|
Ready: ready[m.Name], Total: total[m.Name],
|
||||||
Fallbacks: m.Fallbacks,
|
Fallbacks: m.Fallbacks,
|
||||||
|
Desired: h.Desired, Failures: h.Failures,
|
||||||
|
Parked: h.Parked, Reason: h.Reason, Unplaceable: h.Unplaceable,
|
||||||
|
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
|
||||||
|
External: externalLabel(m), Adapters: m.Adapters,
|
||||||
}
|
}
|
||||||
for _, p := range m.Profiles {
|
for _, p := range m.Profiles {
|
||||||
v.Profiles = append(v.Profiles, p.Name)
|
v.Profiles = append(v.Profiles, p.Name)
|
||||||
}
|
}
|
||||||
|
if len(m.Profiles) > 0 {
|
||||||
|
v.MemoryGB, v.CPUCores = m.Profiles[0].MinMemoryGB, m.Profiles[0].CPUCores
|
||||||
|
}
|
||||||
|
v.MeasuredMemoryGB = float64(measured[m.Name]) / (1 << 30)
|
||||||
|
v.Warnings = reservationWarnings(m, v.MeasuredMemoryGB)
|
||||||
out = append(out, v)
|
out = append(out, v)
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
@@ -471,9 +614,11 @@ func (a *API) Replicas(model string) []ReplicaView {
|
|||||||
ID: rep.ID, Model: rep.Model, Profile: rep.Profile,
|
ID: rep.ID, Model: rep.Model, Profile: rep.Profile,
|
||||||
State: rep.State.String(), Pool: rep.Pool, Fabric: rep.Fabric,
|
State: rep.State.String(), Pool: rep.Pool, Fabric: rep.Fabric,
|
||||||
Endpoint: rep.Endpoint, Err: rep.Err,
|
Endpoint: rep.Endpoint, Err: rep.Err,
|
||||||
Stale: a.profileChanged(rep.Model, rep.Profile, rep.ProfileFP),
|
EngineKey: a.ctl.EngineKey(rep.ID),
|
||||||
AgeSec: int64(time.Since(rep.CreatedAt).Seconds()),
|
Restart: a.ctl.Reconciler().RestartState(rep.ID),
|
||||||
IdleSec: -1,
|
Stale: a.profileChanged(rep.Model, rep.Profile, rep.ProfileFP),
|
||||||
|
AgeSec: int64(time.Since(rep.CreatedAt).Seconds()),
|
||||||
|
IdleSec: -1,
|
||||||
}
|
}
|
||||||
if rep.LastUsedAt.IsZero() {
|
if rep.LastUsedAt.IsZero() {
|
||||||
v.IdleSec = -1
|
v.IdleSec = -1
|
||||||
@@ -513,6 +658,18 @@ type Models interface {
|
|||||||
Add(ctx context.Context, m catalog.Model, by string) error
|
Add(ctx context.Context, m catalog.Model, by string) error
|
||||||
Remove(ctx context.Context, name string) error
|
Remove(ctx context.Context, name string) error
|
||||||
Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error)
|
Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error)
|
||||||
|
Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
|
||||||
|
SetClass(ctx context.Context, name, class, by string) (catalog.Model, error)
|
||||||
|
SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error)
|
||||||
|
SetAdapters(ctx context.Context, name string, adapters []catalog.Adapter, by string) (catalog.Model, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResourcesRequest sets what a model's profiles reserve of a node.
|
||||||
|
type ResourcesRequest struct {
|
||||||
|
// Profile is one profile's name; empty is every profile.
|
||||||
|
Profile string `json:"profile,omitempty"`
|
||||||
|
MemoryGB int `json:"memory_gb"`
|
||||||
|
CPUCores int `json:"cpu_cores"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Usage reports what each group of the tenancy tree spent. The store
|
// Usage reports what each group of the tenancy tree spent. The store
|
||||||
@@ -644,7 +801,7 @@ func (a *API) putCapacity(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
if err := a.capacity.PutCapacity(r.Context(), store.StoredCapacity{
|
if err := a.capacity.PutCapacity(r.Context(), store.StoredCapacity{
|
||||||
Model: c.Model, Profile: c.Profile, Measurement: body,
|
Model: c.Model, Profile: c.Profile, Measurement: body,
|
||||||
MeasuredAt: c.MeasuredAt, MeasuredBy: r.Header.Get("X-Llmgw-Actor"),
|
MeasuredAt: c.MeasuredAt, MeasuredBy: actorOf(r),
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
writeError(w, http.StatusInternalServerError, err.Error())
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
return
|
return
|
||||||
@@ -695,7 +852,7 @@ func (a *API) addModel(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeError(w, http.StatusBadRequest, "the model definition is not valid JSON: "+err.Error())
|
writeError(w, http.StatusBadRequest, "the model definition is not valid JSON: "+err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := a.models.Add(r.Context(), m, r.Header.Get("X-Llmgw-Actor")); err != nil {
|
if err := a.models.Add(r.Context(), m, actorOf(r)); err != nil {
|
||||||
// Every failure here is the definition being wrong, which is the
|
// Every failure here is the definition being wrong, which is the
|
||||||
// caller's to fix — a collision, a missing fallback, a profile that
|
// caller's to fix — a collision, a missing fallback, a profile that
|
||||||
// cannot be placed. None of them are server faults.
|
// cannot be placed. None of them are server faults.
|
||||||
@@ -744,7 +901,7 @@ func (a *API) scaleModel(w http.ResponseWriter, r *http.Request) {
|
|||||||
name := r.PathValue("name")
|
name := r.PathValue("name")
|
||||||
m, err := a.models.Scale(r.Context(), name,
|
m, err := a.models.Scale(r.Context(), name,
|
||||||
catalog.Scaling{Min: req.Min, Max: req.Max, IdleTimeout: req.IdleTimeout},
|
catalog.Scaling{Min: req.Min, Max: req.Max, IdleTimeout: req.IdleTimeout},
|
||||||
r.Header.Get("X-Llmgw-Actor"))
|
actorOf(r))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Like addModel: every refusal here is the request being wrong — a
|
// Like addModel: every refusal here is the request being wrong — a
|
||||||
// model declared in configuration, one that was never added, a max
|
// model declared in configuration, one that was never added, a max
|
||||||
@@ -758,6 +915,26 @@ func (a *API) scaleModel(w http.ResponseWriter, r *http.Request) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// setResources sets a model's reserved memory and cores. Placed replicas keep
|
||||||
|
// what they were started with until they are restarted.
|
||||||
|
func (a *API) setResources(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if a.models == nil {
|
||||||
|
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var req ResourcesRequest
|
||||||
|
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m, err := a.models.Resources(r.Context(), r.PathValue("name"), req.Profile, req.MemoryGB, req.CPUCores, actorOf(r))
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, m)
|
||||||
|
}
|
||||||
|
|
||||||
func (a *API) removeModel(w http.ResponseWriter, r *http.Request) {
|
func (a *API) removeModel(w http.ResponseWriter, r *http.Request) {
|
||||||
if a.models == nil {
|
if a.models == nil {
|
||||||
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||||
@@ -794,6 +971,21 @@ func (a *API) explainPlacement(w http.ResponseWriter, r *http.Request) {
|
|||||||
// It does not check that the model exists; the handler above does that so it
|
// It does not check that the model exists; the handler above does that so it
|
||||||
// can answer 404, while the dashboard only ever asks about models it just read
|
// can answer 404, while the dashboard only ever asks about models it just read
|
||||||
// out of the catalog.
|
// out of the catalog.
|
||||||
|
// Headroom tries placing up to limit more replicas of a model, on a copy of
|
||||||
|
// the inventory, and says how many fit. A limit of zero or less takes 16.
|
||||||
|
func (a *API) Headroom(model string, limit int) (HeadroomView, bool) {
|
||||||
|
if limit <= 0 || limit > 64 {
|
||||||
|
limit = 16
|
||||||
|
}
|
||||||
|
h, ok := a.ctl.Reconciler().Headroom(model, limit)
|
||||||
|
if !ok {
|
||||||
|
return HeadroomView{}, false
|
||||||
|
}
|
||||||
|
return HeadroomView{Model: model, Current: h.Current, More: h.More, Nodes: h.Nodes,
|
||||||
|
Limited: h.Limited, Blocked: h.Blocked, TokPerSec: h.TokPerSec,
|
||||||
|
ByPreempting: h.ByPreempting, Preempts: h.Preempts}, true
|
||||||
|
}
|
||||||
|
|
||||||
func (a *API) Explain(model string) ExplainView {
|
func (a *API) Explain(model string) ExplainView {
|
||||||
v := ExplainView{Model: model}
|
v := ExplainView{Model: model}
|
||||||
for _, rep := range a.ctl.Reconciler().Replicas() {
|
for _, rep := range a.ctl.Reconciler().Replicas() {
|
||||||
@@ -1039,3 +1231,293 @@ func (a *API) Topology(id string) (TopologyView, bool) {
|
|||||||
func (a *API) NVLink() nvlink.Report {
|
func (a *API) NVLink() nvlink.Report {
|
||||||
return nvlink.Analyze(a.ctl.Inventory().Nodes())
|
return nvlink.Analyze(a.ctl.Inventory().Nodes())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RestartRequest names what to restart: replicas by id, every replica of some
|
||||||
|
// models, or — All — every ready replica in the cluster.
|
||||||
|
type RestartRequest struct {
|
||||||
|
Replicas []string `json:"replicas,omitempty"`
|
||||||
|
Models []string `json:"models,omitempty"`
|
||||||
|
All bool `json:"all,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RestartResult lists the replicas queued.
|
||||||
|
type RestartResult struct {
|
||||||
|
Queued []string `json:"queued"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *API) restart(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var req RestartRequest
|
||||||
|
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "the restart request is not valid JSON: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ids := slices.Clone(req.Replicas)
|
||||||
|
for _, rep := range a.ctl.Reconciler().ReadyReplicas() {
|
||||||
|
if req.All || slices.Contains(req.Models, rep.Model) {
|
||||||
|
ids = append(ids, rep.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(ids) == 0 {
|
||||||
|
writeError(w, http.StatusBadRequest, "nothing ready matches; name replicas, models, or all")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := a.ctl.Reconciler().Restart(ids...); err != nil {
|
||||||
|
writeError(w, http.StatusNotFound, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusAccepted, RestartResult{Queued: ids})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *API) retryModel(w http.ResponseWriter, r *http.Request) {
|
||||||
|
name := r.PathValue("name")
|
||||||
|
if _, ok := a.ctl.Catalog().Resolve(name); !ok {
|
||||||
|
writeError(w, http.StatusNotFound, "no model called "+name)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
was := a.ctl.Reconciler().Retry(name)
|
||||||
|
writeJSON(w, http.StatusOK, map[string]bool{"was_parked": was})
|
||||||
|
}
|
||||||
|
|
||||||
|
// requestFinder looks a request up by the id it was given. Asked of the usage
|
||||||
|
// source by type assertion, so one without it answers 501 rather than every
|
||||||
|
// source having to grow the method.
|
||||||
|
type requestFinder interface {
|
||||||
|
UsageByRequest(ctx context.Context, requestID string) (store.Usage, bool, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequestView is what one request went through.
|
||||||
|
type RequestView struct {
|
||||||
|
RequestID string `json:"request_id"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
KeyID string `json:"key_id"`
|
||||||
|
UserID string `json:"user_id,omitempty"`
|
||||||
|
ProjectID string `json:"project_id"`
|
||||||
|
Model string `json:"model"`
|
||||||
|
ServedModel string `json:"served_model"`
|
||||||
|
Replica string `json:"replica,omitempty"`
|
||||||
|
Status int `json:"status"`
|
||||||
|
LatencyMs int64 `json:"latency_ms"`
|
||||||
|
QueueMs int64 `json:"queue_ms"`
|
||||||
|
TTFTMs int64 `json:"ttft_ms"`
|
||||||
|
PromptTokens int64 `json:"prompt_tokens"`
|
||||||
|
CompletionTokens int64 `json:"completion_tokens"`
|
||||||
|
CostMicros int64 `json:"cost_micros"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *API) request(w http.ResponseWriter, r *http.Request) {
|
||||||
|
f, ok := a.usage.(requestFinder)
|
||||||
|
if !ok {
|
||||||
|
writeError(w, http.StatusNotImplemented, "this control plane keeps no usage records")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
u, found, err := f.UsageByRequest(r.Context(), r.PathValue("id"))
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
case !found:
|
||||||
|
writeError(w, http.StatusNotFound, "no request with that id; records are written a few seconds after a request ends")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, RequestView{
|
||||||
|
RequestID: u.RequestID, At: u.At, KeyID: u.KeyID, UserID: u.UserID, ProjectID: u.ProjectID,
|
||||||
|
Model: u.Model, ServedModel: u.ServedModel, Replica: u.Replica, Status: u.Status,
|
||||||
|
LatencyMs: u.LatencyMs, QueueMs: u.QueueMs, TTFTMs: u.TTFTMs,
|
||||||
|
PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetReleases sets where upgrades fetch release binaries from.
|
||||||
|
func (a *API) SetReleases(base string) { a.releases = base }
|
||||||
|
|
||||||
|
// UpgradeRequest names the release a node should run.
|
||||||
|
type UpgradeRequest struct {
|
||||||
|
Version string `json:"version"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *API) upgradeNode(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var req UpgradeRequest
|
||||||
|
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "the upgrade request is not valid JSON: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !release.Tag.MatchString(req.Version) {
|
||||||
|
writeError(w, http.StatusBadRequest, req.Version+" is not a release version")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
u := a.ctl.Upgrader()
|
||||||
|
if u == nil {
|
||||||
|
writeError(w, http.StatusNotImplemented, "this control plane cannot upgrade nodes")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
base := a.releases
|
||||||
|
if base == "" {
|
||||||
|
base = release.DefaultBase
|
||||||
|
}
|
||||||
|
if err := u.Upgrade(r.Context(), r.PathValue("id"), req.Version, base); err != nil {
|
||||||
|
writeError(w, http.StatusBadGateway, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, map[string]string{"node": r.PathValue("id"), "version": req.Version})
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetServe supplies what serves a model from the registry.
|
||||||
|
func (a *API) SetServe(s *modelserve.Server) { a.serve = s }
|
||||||
|
|
||||||
|
func (a *API) serveModel(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if a.serve == nil {
|
||||||
|
writeError(w, http.StatusNotImplemented,
|
||||||
|
"serving from the registry needs a mirror and a store, and this control plane lacks one")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var req modelserve.Request
|
||||||
|
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
req.By = actorOf(r)
|
||||||
|
job, err := a.serve.Start(req)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusAccepted, job)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *API) job(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if a.serve == nil {
|
||||||
|
writeError(w, http.StatusNotFound, "no such job")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
job, ok := a.serve.Get(r.PathValue("id"))
|
||||||
|
if !ok {
|
||||||
|
writeError(w, http.StatusNotFound, "no such job; jobs are kept only until the control plane restarts")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, job)
|
||||||
|
}
|
||||||
|
|
||||||
|
// actorOf is who a change is recorded as made by: what the caller says, since
|
||||||
|
// the admin API answers only this machine and trusts whoever reaches it. Kept
|
||||||
|
// to a short printable name, so a claim cannot pass for more than one — no
|
||||||
|
// line breaks to forge a second log line, no length to bury the rest.
|
||||||
|
func actorOf(r *http.Request) string {
|
||||||
|
claimed := r.Header.Get("X-Llmgw-Actor")
|
||||||
|
var b strings.Builder
|
||||||
|
for _, c := range claimed {
|
||||||
|
if b.Len() >= 64 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if unicode.IsLetter(c) || unicode.IsDigit(c) || strings.ContainsRune("._-@", c) {
|
||||||
|
b.WriteRune(c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if b.Len() == 0 {
|
||||||
|
return "unknown"
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// NodeCPUs is the CPU counts of ready nodes with at least this much memory:
|
||||||
|
// where a replica needing it could go.
|
||||||
|
func (a *API) NodeCPUs(minMemoryGB int) []int {
|
||||||
|
var out []int
|
||||||
|
for _, n := range a.ctl.Inventory().Nodes() {
|
||||||
|
if n.State.String() != "ready" || n.CPUCount == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if n.MemoryTotalMB > 0 && n.MemoryTotalMB/1024 < minMemoryGB {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, n.CPUCount)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// reservationWarnings says what is wrong with what a model reserves.
|
||||||
|
//
|
||||||
|
// Resident memory includes the weights mapped from disk, which the kernel can
|
||||||
|
// take back, so it overstates what a model needs by the part of the weights
|
||||||
|
// not in use. A tenth over is let pass for that; more is a figure too low for
|
||||||
|
// placement to be trusted with.
|
||||||
|
func reservationWarnings(m catalog.Model, measuredGB float64) []string {
|
||||||
|
var out []string
|
||||||
|
for _, p := range m.Profiles {
|
||||||
|
if p.CPUOnly() && p.MinMemoryGB <= 0 {
|
||||||
|
out = append(out, fmt.Sprintf("profile %s declares no memory: placement cannot tell when a node is full "+
|
||||||
|
"(llmgw ctl model resources %s --memory GB)", p.Name, m.Name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(m.Profiles) == 1 && m.Profiles[0].MinMemoryGB > 0 && measuredGB > float64(m.Profiles[0].MinMemoryGB)*1.1 {
|
||||||
|
out = append(out, fmt.Sprintf("uses %.1f GB and reserves %d: raise it, or nodes will be filled past their memory",
|
||||||
|
measuredGB, m.Profiles[0].MinMemoryGB))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// externalLabel is "api.openai.com · gpt-4o-mini" for a model served elsewhere.
|
||||||
|
func externalLabel(m catalog.Model) string {
|
||||||
|
if !m.IsExternal() {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
host := m.External.URL
|
||||||
|
if u, err := url.Parse(host); err == nil && u.Host != "" {
|
||||||
|
host = u.Host
|
||||||
|
}
|
||||||
|
return host + " · " + m.External.Model
|
||||||
|
}
|
||||||
|
|
||||||
|
// AdaptersRequest replaces a model's LoRA adapters with these.
|
||||||
|
type AdaptersRequest struct {
|
||||||
|
Adapters []catalog.Adapter `json:"adapters"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// setAdapters replaces a model's adapters, after reading each new file to
|
||||||
|
// check it is an adapter for the model's architecture: one that is not is a
|
||||||
|
// replica that fails to start, found minutes later on a node.
|
||||||
|
func (a *API) setAdapters(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if a.models == nil {
|
||||||
|
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var req AdaptersRequest
|
||||||
|
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
name := r.PathValue("name")
|
||||||
|
m, ok := a.ctl.Catalog().Resolve(name)
|
||||||
|
if !ok {
|
||||||
|
writeError(w, http.StatusBadRequest, "no model "+name)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
base, baseErr := gguf.KindFromURL(r.Context(), nil, m.Source)
|
||||||
|
have := map[string]bool{}
|
||||||
|
for _, ad := range m.Adapters {
|
||||||
|
have[ad.Source] = true
|
||||||
|
}
|
||||||
|
for _, ad := range req.Adapters {
|
||||||
|
if have[ad.Source] {
|
||||||
|
continue // already checked when it was attached
|
||||||
|
}
|
||||||
|
h, err := gguf.KindFromURL(r.Context(), nil, ad.Source)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
writeError(w, http.StatusBadRequest, fmt.Sprintf("adapter %s: %v", ad.Name, err))
|
||||||
|
return
|
||||||
|
case !h.IsAdapter():
|
||||||
|
writeError(w, http.StatusBadRequest, fmt.Sprintf("adapter %s: %s is a model, not a LoRA adapter", ad.Name, ad.Source))
|
||||||
|
return
|
||||||
|
case baseErr == nil && h.Architecture != base.Architecture:
|
||||||
|
writeError(w, http.StatusBadRequest, fmt.Sprintf("adapter %s is for %s models and %s is %s",
|
||||||
|
ad.Name, h.Architecture, m.Name, base.Architecture))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out, err := a.models.SetAdapters(r.Context(), m.Name, req.Adapters, actorOf(r))
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, AdaptersRequest{Adapters: out.Adapters})
|
||||||
|
}
|
||||||
@@ -258,6 +258,25 @@ type fakeModels struct {
|
|||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f *fakeModels) SetAdapters(_ context.Context, name string, as []catalog.Adapter, _ string) (catalog.Model, error) {
|
||||||
|
return catalog.Model{Name: name, Adapters: as}, f.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeModels) SetOwner(_ context.Context, name, owner, _ string) (catalog.Model, error) {
|
||||||
|
return catalog.Model{Name: name, Owner: owner}, f.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeModels) SetClass(_ context.Context, name, class, _ string) (catalog.Model, error) {
|
||||||
|
return catalog.Model{Name: name, Class: class}, f.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeModels) Resources(_ context.Context, name, profile string, mem, cores int, _ string) (catalog.Model, error) {
|
||||||
|
if f.err != nil {
|
||||||
|
return catalog.Model{}, f.err
|
||||||
|
}
|
||||||
|
return catalog.Model{Name: name, Profiles: []catalog.Profile{{Name: "p", MinMemoryGB: mem, CPUCores: cores}}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (f *fakeModels) Scale(_ context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error) {
|
func (f *fakeModels) Scale(_ context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error) {
|
||||||
if f.err != nil {
|
if f.err != nil {
|
||||||
return catalog.Model{}, f.err
|
return catalog.Model{}, f.err
|
||||||
|
|||||||
@@ -0,0 +1,285 @@
|
|||||||
|
package adminapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Quotas is where capacity quotas and the tenancy tree they name are kept.
|
||||||
|
// The store implements it.
|
||||||
|
type Quotas interface {
|
||||||
|
CapacityQuotas(ctx context.Context) ([]store.CapacityQuota, error)
|
||||||
|
PutCapacityQuota(ctx context.Context, q store.CapacityQuota) error
|
||||||
|
DeleteCapacityQuota(ctx context.Context, scope store.Scope, id string) error
|
||||||
|
ScopeParents(ctx context.Context) (map[string][]string, error)
|
||||||
|
AllOrgs(ctx context.Context) ([]store.Org, error)
|
||||||
|
AllTeams(ctx context.Context) ([]store.TeamPath, error)
|
||||||
|
AllProjects(ctx context.Context) ([]store.ProjectPath, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetQuotas supplies the quota store, which mounts the quota endpoints'
|
||||||
|
// behaviour; without it they answer 501.
|
||||||
|
func (a *API) SetQuotas(q Quotas) { a.quotas = q }
|
||||||
|
|
||||||
|
// QuotaView is one scope's quota with what it holds now.
|
||||||
|
type QuotaView struct {
|
||||||
|
// Scope is team:ID; Name is how people know it, "Acme / ml".
|
||||||
|
Scope string `json:"scope"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Guarantee store.Resources `json:"guarantee"`
|
||||||
|
Limit store.Resources `json:"limit"`
|
||||||
|
Holds quota.Amount `json:"holds"`
|
||||||
|
UpdatedBy string `json:"updated_by,omitempty"`
|
||||||
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// QuotaRequest sets a scope's quota. Scope may be written with an ID or a
|
||||||
|
// name: org:Acme, team:Acme/ml, project:Acme/ml/training.
|
||||||
|
type QuotaRequest struct {
|
||||||
|
Scope string `json:"scope"`
|
||||||
|
Guarantee store.Resources `json:"guarantee"`
|
||||||
|
Limit store.Resources `json:"limit"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *API) registerQuotas(mux *http.ServeMux) {
|
||||||
|
mux.HandleFunc("GET /api/v1/quotas", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if a.quotas == nil {
|
||||||
|
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
views, err := a.QuotaViews(r.Context())
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, views)
|
||||||
|
})
|
||||||
|
mux.HandleFunc("PUT /api/v1/quotas", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if a.quotas == nil {
|
||||||
|
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var req QuotaRequest
|
||||||
|
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<14)).Decode(&req); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
key, err := a.SetQuota(r.Context(), req, actorOf(r))
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, map[string]string{"scope": key})
|
||||||
|
})
|
||||||
|
mux.HandleFunc("DELETE /api/v1/quotas/{scope}", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if a.quotas == nil {
|
||||||
|
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
key, err := a.ClearQuota(r.Context(), r.PathValue("scope"))
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, map[string]string{"removed": key})
|
||||||
|
})
|
||||||
|
mux.HandleFunc("POST /api/v1/models/{name}/owner", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if a.models == nil || a.quotas == nil {
|
||||||
|
writeError(w, http.StatusNotImplemented, "this control plane has no store")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var req struct {
|
||||||
|
Owner string `json:"owner"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
owner := ""
|
||||||
|
if req.Owner != "" {
|
||||||
|
scope, id, _, err := a.resolveScope(r.Context(), req.Owner)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
owner = quota.Key(scope, id)
|
||||||
|
}
|
||||||
|
m, err := a.models.SetOwner(r.Context(), r.PathValue("name"), owner, actorOf(r))
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, map[string]string{"model": m.Name, "owner": m.Owner})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// QuotaViews lists every quota with what its scope holds now.
|
||||||
|
func (a *API) QuotaViews(ctx context.Context) ([]QuotaView, error) {
|
||||||
|
if a.quotas == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
qs, err := a.quotas.CapacityQuotas(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
parents, err := a.quotas.ScopeParents(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
names, err := a.scopeNames(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
usage := quota.New(qs, parents).Usage(a.ctl.Reconciler().Holdings())
|
||||||
|
out := make([]QuotaView, 0, len(qs))
|
||||||
|
for _, q := range qs {
|
||||||
|
key := quota.Key(q.Scope, q.ScopeID)
|
||||||
|
out = append(out, QuotaView{Scope: key, Name: names[key], Guarantee: q.Guarantee, Limit: q.Limit,
|
||||||
|
Holds: usage[key], UpdatedBy: q.UpdatedBy, UpdatedAt: q.UpdatedAt})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// scopeNames maps org:ID, team:ID and project:ID to their paths.
|
||||||
|
func (a *API) scopeNames(ctx context.Context) (map[string]string, error) {
|
||||||
|
out := map[string]string{}
|
||||||
|
orgs, err := a.quotas.AllOrgs(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, o := range orgs {
|
||||||
|
out["org:"+o.ID] = o.Name
|
||||||
|
}
|
||||||
|
teams, err := a.quotas.AllTeams(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, t := range teams {
|
||||||
|
out["team:"+t.ID] = t.OrgName + " / " + t.Name
|
||||||
|
}
|
||||||
|
projects, err := a.quotas.AllProjects(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, p := range projects {
|
||||||
|
out["project:"+p.ID] = p.Path
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveScope reads org:X, team:X or project:X, where X is an ID or the
|
||||||
|
// scope's path — Acme, Acme/ml, Acme/ml/training — and returns its ID and
|
||||||
|
// path. A path that could mean two scopes is refused rather than guessed.
|
||||||
|
func (a *API) resolveScope(ctx context.Context, s string) (store.Scope, string, string, error) {
|
||||||
|
kind, what, ok := strings.Cut(strings.TrimSpace(s), ":")
|
||||||
|
scope := store.Scope(kind)
|
||||||
|
if !ok || what == "" || (scope != store.ScopeOrg && scope != store.ScopeTeam && scope != store.ScopeProject) {
|
||||||
|
return "", "", "", fmt.Errorf("%q is not org:NAME, team:ORG/TEAM or project:ORG/TEAM/PROJECT", s)
|
||||||
|
}
|
||||||
|
names, err := a.scopeNames(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", "", err
|
||||||
|
}
|
||||||
|
norm := func(p string) string {
|
||||||
|
parts := strings.Split(p, "/")
|
||||||
|
for i := range parts {
|
||||||
|
parts[i] = strings.ToLower(strings.TrimSpace(parts[i]))
|
||||||
|
}
|
||||||
|
return strings.Join(parts, "/")
|
||||||
|
}
|
||||||
|
if name, ok := names[kind+":"+what]; ok {
|
||||||
|
return scope, what, name, nil
|
||||||
|
}
|
||||||
|
match := func(same func(name string) bool) []string {
|
||||||
|
var found []string
|
||||||
|
for key, name := range names {
|
||||||
|
if strings.HasPrefix(key, kind+":") && same(name) {
|
||||||
|
found = append(found, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return found
|
||||||
|
}
|
||||||
|
found := match(func(name string) bool { return norm(name) == norm(what) })
|
||||||
|
if len(found) == 0 {
|
||||||
|
// Its own name alone — ml rather than Acme/ml — when that is unique.
|
||||||
|
found = match(func(name string) bool {
|
||||||
|
parts := strings.Split(norm(name), "/")
|
||||||
|
return parts[len(parts)-1] == norm(what)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
switch len(found) {
|
||||||
|
case 0:
|
||||||
|
return "", "", "", fmt.Errorf("there is no %s %q", kind, what)
|
||||||
|
case 1:
|
||||||
|
_, id, _ := strings.Cut(found[0], ":")
|
||||||
|
return scope, id, names[found[0]], nil
|
||||||
|
}
|
||||||
|
return "", "", "", fmt.Errorf("%q could be any of %d %ss; give its ID", what, len(found), kind)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetQuota sets a scope's capacity quota, named by path or ID, and returns
|
||||||
|
// the scope as org:ID.
|
||||||
|
func (a *API) SetQuota(ctx context.Context, req QuotaRequest, by string) (string, error) {
|
||||||
|
if a.quotas == nil {
|
||||||
|
return "", fmt.Errorf("this control plane has no store")
|
||||||
|
}
|
||||||
|
scope, id, _, err := a.resolveScope(ctx, req.Scope)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if !req.Guarantee.Any() && !req.Limit.Any() {
|
||||||
|
return "", fmt.Errorf("a quota needs a guarantee or a limit; to remove one, clear it")
|
||||||
|
}
|
||||||
|
q := store.CapacityQuota{Scope: scope, ScopeID: id, Guarantee: req.Guarantee, Limit: req.Limit, UpdatedBy: by}
|
||||||
|
if err := a.quotas.PutCapacityQuota(ctx, q); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return quota.Key(scope, id), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ClearQuota removes a scope's capacity quota.
|
||||||
|
func (a *API) ClearQuota(ctx context.Context, scope string) (string, error) {
|
||||||
|
if a.quotas == nil {
|
||||||
|
return "", fmt.Errorf("this control plane has no store")
|
||||||
|
}
|
||||||
|
s, id, _, err := a.resolveScope(ctx, scope)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := a.quotas.DeleteCapacityQuota(ctx, s, id); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return quota.Key(s, id), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScopeName is a part of the tenancy tree: org:ID and its path.
|
||||||
|
type ScopeName struct {
|
||||||
|
Key string `json:"scope"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scopes lists every organization, team and project, by path.
|
||||||
|
func (a *API) Scopes(ctx context.Context) ([]ScopeName, error) {
|
||||||
|
if a.quotas == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
names, err := a.scopeNames(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := make([]ScopeName, 0, len(names))
|
||||||
|
for k, n := range names {
|
||||||
|
out = append(out, ScopeName{Key: k, Name: n})
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
package adminapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeQuotaStore is Acme with teams ml and web, and Scania with another ml.
|
||||||
|
type fakeQuotaStore struct {
|
||||||
|
quotas []store.CapacityQuota
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeQuotaStore) CapacityQuotas(context.Context) ([]store.CapacityQuota, error) {
|
||||||
|
return f.quotas, nil
|
||||||
|
}
|
||||||
|
func (f *fakeQuotaStore) PutCapacityQuota(_ context.Context, q store.CapacityQuota) error {
|
||||||
|
for i := range f.quotas {
|
||||||
|
if f.quotas[i].Scope == q.Scope && f.quotas[i].ScopeID == q.ScopeID {
|
||||||
|
f.quotas[i] = q
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f.quotas = append(f.quotas, q)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (f *fakeQuotaStore) DeleteCapacityQuota(_ context.Context, s store.Scope, id string) error {
|
||||||
|
for i, q := range f.quotas {
|
||||||
|
if q.Scope == s && q.ScopeID == id {
|
||||||
|
f.quotas = append(f.quotas[:i], f.quotas[i+1:]...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (f *fakeQuotaStore) ScopeParents(context.Context) (map[string][]string, error) {
|
||||||
|
return map[string][]string{"team:t-ml": {"org:o-acme"}, "team:t-web": {"org:o-acme"},
|
||||||
|
"team:t-sml": {"org:o-scania"}}, nil
|
||||||
|
}
|
||||||
|
func (f *fakeQuotaStore) AllOrgs(context.Context) ([]store.Org, error) {
|
||||||
|
return []store.Org{{ID: "o-acme", Name: "Acme"}, {ID: "o-scania", Name: "Scania"}}, nil
|
||||||
|
}
|
||||||
|
func (f *fakeQuotaStore) AllTeams(context.Context) ([]store.TeamPath, error) {
|
||||||
|
return []store.TeamPath{{ID: "t-ml", Name: "ml", OrgID: "o-acme", OrgName: "Acme"},
|
||||||
|
{ID: "t-web", Name: "web", OrgID: "o-acme", OrgName: "Acme"},
|
||||||
|
{ID: "t-sml", Name: "ml", OrgID: "o-scania", OrgName: "Scania"}}, nil
|
||||||
|
}
|
||||||
|
func (f *fakeQuotaStore) AllProjects(context.Context) ([]store.ProjectPath, error) { return nil, nil }
|
||||||
|
|
||||||
|
func TestAScopeIsNamedByPathOrIDAndNeverGuessed(t *testing.T) {
|
||||||
|
a, _ := setup(t, testModels())
|
||||||
|
a.SetQuotas(&fakeQuotaStore{})
|
||||||
|
for in, want := range map[string]string{
|
||||||
|
"org:Acme": "o-acme", "org:o-acme": "o-acme", "team:Acme/ml": "t-ml",
|
||||||
|
"team: acme / ML ": "t-ml", "team:Scania/ml": "t-sml", "team:web": "t-web",
|
||||||
|
} {
|
||||||
|
_, id, _, err := a.resolveScope(t.Context(), in)
|
||||||
|
if err != nil || id != want {
|
||||||
|
t.Errorf("%q: %q %v, expected %q", in, id, err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, bad := range []string{"team:ml", "team:Acme/nope", "group:Acme", "Acme", "org:"} {
|
||||||
|
if _, _, _, err := a.resolveScope(t.Context(), bad); err == nil {
|
||||||
|
t.Errorf("%q was resolved", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, _, _, err := a.resolveScope(t.Context(), "team:ml"); err == nil || !strings.Contains(err.Error(), "could be any of 2") {
|
||||||
|
t.Errorf("an ambiguous name should say so: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAQuotaIsSetListedAndCleared(t *testing.T) {
|
||||||
|
a, _ := setup(t, testModels())
|
||||||
|
qs := &fakeQuotaStore{}
|
||||||
|
a.SetQuotas(qs)
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
a.Register(mux)
|
||||||
|
call := func(method, path, body string) *httptest.ResponseRecorder {
|
||||||
|
req := httptest.NewRequest(method, path, strings.NewReader(body))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("X-Llmgw-Actor", "ops")
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
mux.ServeHTTP(rec, req)
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
if rec := call("PUT", "/api/v1/quotas", `{"scope":"team:Acme/ml","guarantee":{"gpus":8},"limit":{"gpus":16}}`); rec.Code != 200 {
|
||||||
|
t.Fatalf("set: %d %s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
if len(qs.quotas) != 1 || qs.quotas[0].ScopeID != "t-ml" || *qs.quotas[0].Guarantee.GPUs != 8 || qs.quotas[0].UpdatedBy != "ops" {
|
||||||
|
t.Errorf("stored %+v", qs.quotas)
|
||||||
|
}
|
||||||
|
rec := call("GET", "/api/v1/quotas", "")
|
||||||
|
var views []QuotaView
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &views); err != nil || len(views) != 1 || views[0].Name != "Acme / ml" {
|
||||||
|
t.Errorf("list %d %s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
if rec := call("PUT", "/api/v1/quotas", `{"scope":"team:Acme/ml"}`); rec.Code != 400 {
|
||||||
|
t.Errorf("an empty quota was accepted: %d", rec.Code)
|
||||||
|
}
|
||||||
|
if rec := call("DELETE", "/api/v1/quotas/team:Acme%2Fml", ""); rec.Code != 200 || len(qs.quotas) != 0 {
|
||||||
|
t.Errorf("clear: %d %s %+v", rec.Code, rec.Body, qs.quotas)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,6 +25,9 @@ type Summary struct {
|
|||||||
// Unplaceable names models the scheduler currently cannot satisfy. This is
|
// Unplaceable names models the scheduler currently cannot satisfy. This is
|
||||||
// the line that turns "something is wrong" into a place to look.
|
// the line that turns "something is wrong" into a place to look.
|
||||||
Unplaceable []string `json:"unplaceable,omitempty"`
|
Unplaceable []string `json:"unplaceable,omitempty"`
|
||||||
|
// Parked are models that failed every time they started and are no
|
||||||
|
// longer retried, with the last error.
|
||||||
|
Parked []ParkedModel `json:"parked,omitempty"`
|
||||||
// IdleReplicas are serving replicas nothing has asked for in a while, which
|
// IdleReplicas are serving replicas nothing has asked for in a while, which
|
||||||
// on a fleet with more models than GPUs is the first place to find room.
|
// on a fleet with more models than GPUs is the first place to find room.
|
||||||
IdleReplicas []IdleReplica `json:"idle_replicas,omitempty"`
|
IdleReplicas []IdleReplica `json:"idle_replicas,omitempty"`
|
||||||
@@ -201,17 +204,30 @@ func (a *API) Summary() Summary {
|
|||||||
|
|
||||||
for _, m := range a.Models() {
|
for _, m := range a.Models() {
|
||||||
s.Models.Total++
|
s.Models.Total++
|
||||||
if m.Ready >= m.Min {
|
if m.Parked {
|
||||||
|
s.Parked = append(s.Parked, ParkedModel{Model: m.Name, Failures: m.Failures, Reason: m.Reason})
|
||||||
|
}
|
||||||
|
// Against what it should have now, not its minimum: an on-demand
|
||||||
|
// model's minimum is 0, so one that was asked for and fitted nowhere
|
||||||
|
// used to count as satisfied.
|
||||||
|
if m.Ready >= m.Desired && !m.Parked {
|
||||||
s.Models.Satisfied++
|
s.Models.Satisfied++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
s.Models.Short++
|
s.Models.Short++
|
||||||
// Short of replicas and unable to place more is the actionable case;
|
// Short of replicas and unable to place more is the actionable case;
|
||||||
// short and still starting is just a cluster doing its job.
|
// short and still starting is just a cluster doing its job.
|
||||||
if m.Total <= m.Ready {
|
if m.Unplaceable || (m.Total <= m.Ready && !m.Parked) {
|
||||||
s.Unplaceable = append(s.Unplaceable, m.Name)
|
s.Unplaceable = append(s.Unplaceable, m.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sort.Strings(s.Unplaceable)
|
sort.Strings(s.Unplaceable)
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ParkedModel is a model no longer retried, and why.
|
||||||
|
type ParkedModel struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
Failures int `json:"failures"`
|
||||||
|
Reason string `json:"reason"`
|
||||||
|
}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
package adminapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TargetView is how many replicas a model needs to serve a throughput, from
|
||||||
|
// what one replica was measured to serve, and whether the cluster has room.
|
||||||
|
type TargetView struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
// TokPerSec is what was asked for; PerReplica what one replica was
|
||||||
|
// measured to serve at its SLO, on Profile.
|
||||||
|
TokPerSec float64 `json:"tok_per_sec"`
|
||||||
|
PerReplica float64 `json:"per_replica_tok_per_sec"`
|
||||||
|
Profile string `json:"profile"`
|
||||||
|
// Needed replicas in all; Current hold capacity now; More is how many of
|
||||||
|
// Needed are not running, and Room how many more fit.
|
||||||
|
Needed int `json:"needed"`
|
||||||
|
Current int `json:"current"`
|
||||||
|
More int `json:"more"`
|
||||||
|
Room int `json:"room"`
|
||||||
|
Fits bool `json:"fits"`
|
||||||
|
Blocked []string `json:"blocked,omitempty"`
|
||||||
|
// Preempts is whose replicas would be stopped to make the room, when
|
||||||
|
// free room alone is not enough; Room counts that room too.
|
||||||
|
Preempts []string `json:"preempts,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Target sizes a model for a throughput.
|
||||||
|
func (a *API) Target(model string, tokPerSec float64) (TargetView, error) {
|
||||||
|
meas, profile, ok := a.ctl.Reconciler().MeasuredFor(model)
|
||||||
|
if !ok || meas.TokPerSec <= 0 {
|
||||||
|
return TargetView{}, fmt.Errorf("%s has not been measured; run `llmgw ctl bench %s --save` first", model, model)
|
||||||
|
}
|
||||||
|
v := TargetView{Model: model, TokPerSec: tokPerSec, PerReplica: meas.TokPerSec, Profile: profile,
|
||||||
|
Needed: max(int(math.Ceil(tokPerSec/meas.TokPerSec)), 1)}
|
||||||
|
h, ok := a.Headroom(model, v.Needed)
|
||||||
|
if !ok {
|
||||||
|
return TargetView{}, fmt.Errorf("no model %q", model)
|
||||||
|
}
|
||||||
|
v.Current = h.Current
|
||||||
|
v.More = max(v.Needed-h.Current, 0)
|
||||||
|
v.Room = h.More + h.ByPreempting
|
||||||
|
v.Fits = v.More <= v.Room || h.Limited
|
||||||
|
if v.More > h.More {
|
||||||
|
v.Preempts = h.Preempts
|
||||||
|
}
|
||||||
|
if !v.Fits {
|
||||||
|
v.Blocked = h.Blocked
|
||||||
|
}
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *API) target(w http.ResponseWriter, r *http.Request) {
|
||||||
|
tps, err := strconv.ParseFloat(r.URL.Query().Get("tokens_per_sec"), 64)
|
||||||
|
if err != nil || tps <= 0 {
|
||||||
|
writeError(w, http.StatusBadRequest, "tokens_per_sec must be a positive number")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
v, err := a.Target(r.PathValue("name"), tps)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, v)
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
package adminapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestATargetIsSizedFromTheMeasurementAndTheRoom(t *testing.T) {
|
||||||
|
// Three 8-GPU nodes; one tp8 replica was measured at 500 tokens/s.
|
||||||
|
a, ctl := setup(t, testModels(), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
||||||
|
if _, err := a.Target("llama", 1000); err == nil || !strings.Contains(err.Error(), "ctl bench llama --save") {
|
||||||
|
t.Errorf("an unmeasured model should say how to measure it: %v", err)
|
||||||
|
}
|
||||||
|
ctl.Reconciler().SetMeasurements(func() map[string]reconciler.Measured {
|
||||||
|
return map[string]reconciler.Measured{"llama/tp8": {MaxConcurrency: 16, TokPerSec: 500}}
|
||||||
|
})
|
||||||
|
v, err := a.Target("llama", 1200)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if v.Needed != 3 || v.PerReplica != 500 || v.Profile != "tp8" || v.More != 3 || !v.Fits {
|
||||||
|
t.Errorf("1200 tok/s at 500 each is 3 replicas on 3 empty nodes: %+v", v)
|
||||||
|
}
|
||||||
|
v, _ = a.Target("llama", 2000)
|
||||||
|
if v.Needed != 4 || v.Fits || v.Room != 3 || len(v.Blocked) == 0 {
|
||||||
|
t.Errorf("a fourth does not fit on three nodes, and should say why: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -31,6 +31,11 @@ func (a *API) Orphans() []OrphanView {
|
|||||||
var wanted []string
|
var wanted []string
|
||||||
for _, m := range a.ctl.Catalog().Models() {
|
for _, m := range a.ctl.Catalog().Models() {
|
||||||
wanted = append(wanted, m.Source)
|
wanted = append(wanted, m.Source)
|
||||||
|
// A model's adapters are its weights too; collecting them would
|
||||||
|
// leave replicas that cannot restart.
|
||||||
|
for _, ad := range m.Adapters {
|
||||||
|
wanted = append(wanted, ad.Source)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
out := make([]OrphanView, 0)
|
out := make([]OrphanView, 0)
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ package agentsvc
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"maps"
|
"maps"
|
||||||
|
"runtime"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||||
@@ -27,6 +28,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
|
|||||||
EngineVersions: r.GetEngineVersions(),
|
EngineVersions: r.GetEngineVersions(),
|
||||||
EnginesProbed: r.GetEnginesProbed(),
|
EnginesProbed: r.GetEnginesProbed(),
|
||||||
MemoryTotalMB: int(r.GetMemoryTotalMb()),
|
MemoryTotalMB: int(r.GetMemoryTotalMb()),
|
||||||
|
CPUCount: int(r.GetCpuCount()),
|
||||||
}
|
}
|
||||||
if h := r.GetHost(); h != nil {
|
if h := r.GetHost(); h != nil {
|
||||||
n.Host = discovery.Host{
|
n.Host = discovery.Host{
|
||||||
@@ -90,6 +92,7 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
|
|||||||
NodeId: nodeID, Hostname: hostname, MgmtAddr: mgmtAddr,
|
NodeId: nodeID, Hostname: hostname, MgmtAddr: mgmtAddr,
|
||||||
Labels: labels, AgentVersion: agentVersion,
|
Labels: labels, AgentVersion: agentVersion,
|
||||||
MemoryTotalMb: int64(inv.MemoryTotalMB),
|
MemoryTotalMb: int64(inv.MemoryTotalMB),
|
||||||
|
CpuCount: int32(runtime.NumCPU()),
|
||||||
}
|
}
|
||||||
for _, w := range inv.Weights {
|
for _, w := range inv.Weights {
|
||||||
r.Weights = append(r.Weights, &llmgwv1.LocalWeights{
|
r.Weights = append(r.Weights, &llmgwv1.LocalWeights{
|
||||||
@@ -186,11 +189,14 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
|
|||||||
GpusPerNode: int32(p.GPUsPerNode), Nodes: int32(p.Nodes),
|
GpusPerNode: int32(p.GPUsPerNode), Nodes: int32(p.Nodes),
|
||||||
Args: p.Args, Env: p.Env,
|
Args: p.Args, Env: p.Env,
|
||||||
ModelPath: p.ModelPath, ServedModelName: p.ServedModelName,
|
ModelPath: p.ModelPath, ServedModelName: p.ServedModelName,
|
||||||
|
Adapters: adaptersToWire(p.Adapters),
|
||||||
},
|
},
|
||||||
StartupTimeoutMs: cmd.StartupTimeout.Milliseconds(),
|
StartupTimeoutMs: cmd.StartupTimeout.Milliseconds(),
|
||||||
Pool: cmd.Pool,
|
Pool: cmd.Pool,
|
||||||
Fabric: cmd.Fabric,
|
Fabric: cmd.Fabric,
|
||||||
ProfileFp: cmd.ProfileFP,
|
ProfileFp: cmd.ProfileFP,
|
||||||
|
ApiKey: cmd.Spec.APIKey,
|
||||||
|
CpuThreads: int32(cmd.Spec.Threads),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -211,12 +217,15 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration
|
|||||||
Pool: m.GetPool(),
|
Pool: m.GetPool(),
|
||||||
Fabric: m.GetFabric(),
|
Fabric: m.GetFabric(),
|
||||||
ProfileFP: m.GetProfileFp(),
|
ProfileFP: m.GetProfileFp(),
|
||||||
|
APIKey: m.GetApiKey(),
|
||||||
|
Threads: int(m.GetCpuThreads()),
|
||||||
Profile: engines.Profile{
|
Profile: engines.Profile{
|
||||||
Name: p.GetName(), Engine: p.GetEngine(), GPUClass: p.GetGpuClass(),
|
Name: p.GetName(), Engine: p.GetEngine(), GPUClass: p.GetGpuClass(),
|
||||||
TP: int(p.GetTp()), PP: int(p.GetPp()),
|
TP: int(p.GetTp()), PP: int(p.GetPp()),
|
||||||
GPUsPerNode: int(p.GetGpusPerNode()), Nodes: int(p.GetNodes()),
|
GPUsPerNode: int(p.GetGpusPerNode()), Nodes: int(p.GetNodes()),
|
||||||
Args: p.GetArgs(), Env: p.GetEnv(),
|
Args: p.GetArgs(), Env: p.GetEnv(),
|
||||||
ModelPath: p.GetModelPath(), ServedModelName: p.GetServedModelName(),
|
ModelPath: p.GetModelPath(), ServedModelName: p.GetServedModelName(),
|
||||||
|
Adapters: adaptersFromWire(p.GetAdapters()),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
return spec, time.Duration(m.GetStartupTimeoutMs()) * time.Millisecond
|
return spec, time.Duration(m.GetStartupTimeoutMs()) * time.Millisecond
|
||||||
@@ -285,3 +294,19 @@ func fabricFrom(state, status, cluster, clique, bandwidth string) *discovery.GPU
|
|||||||
}
|
}
|
||||||
return &discovery.GPUFabric{State: state, Status: status, ClusterUUID: cluster, CliqueID: clique, Bandwidth: bandwidth}
|
return &discovery.GPUFabric{State: state, Status: status, ClusterUUID: cluster, CliqueID: clique, Bandwidth: bandwidth}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func adaptersToWire(as []engines.Adapter) []*llmgwv1.LoraAdapter {
|
||||||
|
var out []*llmgwv1.LoraAdapter
|
||||||
|
for _, a := range as {
|
||||||
|
out = append(out, &llmgwv1.LoraAdapter{Name: a.Name, Source: a.Path})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func adaptersFromWire(as []*llmgwv1.LoraAdapter) []engines.Adapter {
|
||||||
|
var out []engines.Adapter
|
||||||
|
for _, a := range as {
|
||||||
|
out = append(out, engines.Adapter{Name: a.GetName(), Path: a.GetSource()})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
package agentsvc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAdaptersTravelToTheAgentInOrder(t *testing.T) {
|
||||||
|
cmd := reconciler.ShardCommand{ReplicaID: "r", ShardID: "s", Spec: engines.ShardSpec{
|
||||||
|
Profile: engines.Profile{Name: "cpu", Engine: "llamacpp", ModelPath: "/m.gguf",
|
||||||
|
Adapters: []engines.Adapter{{Name: "sql", Path: "http://mirror/sql.gguf"}, {Name: "docs", Path: "http://mirror/docs.gguf"}}},
|
||||||
|
}}
|
||||||
|
spec, _ := ShardSpecFromStart(StartShardFromCommand(cmd))
|
||||||
|
got := spec.Profile.Adapters
|
||||||
|
if len(got) != 2 || got[0].Name != "sql" || got[1].Path != "http://mirror/docs.gguf" {
|
||||||
|
t.Errorf("adapters %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -264,3 +264,22 @@ func TestAGPUFallingOutOfItsDomainTakesTheNodeOut(t *testing.T) {
|
|||||||
t.Errorf("imex = %+v", n.IMEX)
|
t.Errorf("imex = %+v", n.IMEX)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAnUpgradeWaitsForTheAgentsAnswer(t *testing.T) {
|
||||||
|
s := newTestService()
|
||||||
|
c := &conn{nodeID: "n1", send: make(chan *llmgwv1.ControlMessage, 4), closed: make(chan struct{})}
|
||||||
|
s.conns["n1"] = c
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- s.Upgrade(t.Context(), "n1", "v1.2.3", "https://r") }()
|
||||||
|
|
||||||
|
req := (<-c.send).GetUpgrade()
|
||||||
|
if req.GetVersion() != "v1.2.3" || req.GetReleases() != "https://r" {
|
||||||
|
t.Fatalf("sent %v", req)
|
||||||
|
}
|
||||||
|
s.handle("n1", &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: &llmgwv1.UpgradeReply{
|
||||||
|
RequestId: req.GetRequestId(), Version: "v1.2.3", Error: "checksum mismatch",
|
||||||
|
}}})
|
||||||
|
if err := <-done; err == nil || err.Error() != "checksum mismatch" {
|
||||||
|
t.Errorf("the agent's refusal was not passed on: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -60,6 +60,8 @@ type Service struct {
|
|||||||
logMu sync.Mutex
|
logMu sync.Mutex
|
||||||
logWaits map[string]chan *llmgwv1.LogsReply
|
logWaits map[string]chan *llmgwv1.LogsReply
|
||||||
logSeq uint64
|
logSeq uint64
|
||||||
|
// upgradeWaits are the upgrade requests awaiting a reply; guarded by logMu.
|
||||||
|
upgradeWaits map[string]chan *llmgwv1.UpgradeReply
|
||||||
// scanWaits are the fabric scans awaiting a reply, by request id.
|
// scanWaits are the fabric scans awaiting a reply, by request id.
|
||||||
scanWaits map[string]chan *llmgwv1.FabricScanReply
|
scanWaits map[string]chan *llmgwv1.FabricScanReply
|
||||||
testWaits map[string]chan *llmgwv1.FabricTestReply
|
testWaits map[string]chan *llmgwv1.FabricTestReply
|
||||||
@@ -309,6 +311,15 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
|
|||||||
ch <- m.FabricScan
|
ch <- m.FabricScan
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case *llmgwv1.AgentMessage_Upgrade:
|
||||||
|
s.logMu.Lock()
|
||||||
|
ch, ok := s.upgradeWaits[m.Upgrade.GetRequestId()]
|
||||||
|
delete(s.upgradeWaits, m.Upgrade.GetRequestId())
|
||||||
|
s.logMu.Unlock()
|
||||||
|
if ok {
|
||||||
|
ch <- m.Upgrade
|
||||||
|
}
|
||||||
|
|
||||||
case *llmgwv1.AgentMessage_Logs:
|
case *llmgwv1.AgentMessage_Logs:
|
||||||
s.logMu.Lock()
|
s.logMu.Lock()
|
||||||
ch, ok := s.logWaits[m.Logs.GetRequestId()]
|
ch, ok := s.logWaits[m.Logs.GetRequestId()]
|
||||||
@@ -334,8 +345,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
|||||||
problems := hb.GetNewProblems()
|
problems := hb.GetNewProblems()
|
||||||
storage := hb.GetStorage()
|
storage := hb.GetStorage()
|
||||||
memMB := int(hb.GetMemoryTotalMb())
|
memMB := int(hb.GetMemoryTotalMb())
|
||||||
|
cpus := int(hb.GetCpuCount())
|
||||||
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 &&
|
if len(used) == 0 && len(problems) == 0 && len(storage) == 0 &&
|
||||||
!hb.GetWeightsChanged() && memMB == 0 && !hb.GetEnginesChanged() &&
|
!hb.GetWeightsChanged() && memMB == 0 && cpus == 0 && !hb.GetEnginesChanged() &&
|
||||||
!hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 &&
|
!hb.GetProvisionChanged() && hb.GetLoad() == nil && len(hb.GetIbPorts()) == 0 &&
|
||||||
!hb.GetFabricsChanged() && len(hb.GetGpuFabric()) == 0 && hb.GetImex() == nil {
|
!hb.GetFabricsChanged() && len(hb.GetGpuFabric()) == 0 && hb.GetImex() == nil {
|
||||||
return
|
return
|
||||||
@@ -395,6 +407,12 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
|||||||
"from_mb", n.MemoryTotalMB, "to_mb", memMB)
|
"from_mb", n.MemoryTotalMB, "to_mb", memMB)
|
||||||
n.MemoryTotalMB = memMB
|
n.MemoryTotalMB = memMB
|
||||||
}
|
}
|
||||||
|
if cpus > 0 && cpus != n.CPUCount {
|
||||||
|
if n.CPUCount > 0 {
|
||||||
|
s.log.Info("node CPUs changed", "node", nodeID, "from", n.CPUCount, "to", cpus)
|
||||||
|
}
|
||||||
|
n.CPUCount = cpus
|
||||||
|
}
|
||||||
if hb.GetEnginesChanged() {
|
if hb.GetEnginesChanged() {
|
||||||
vers := maps.Clone(hb.GetEngineVersions())
|
vers := maps.Clone(hb.GetEngineVersions())
|
||||||
if !maps.Equal(vers, n.EngineVersions) {
|
if !maps.Equal(vers, n.EngineVersions) {
|
||||||
@@ -527,6 +545,13 @@ func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load {
|
|||||||
CPUCores: sh.GetCpuCores(), RSSBytes: sh.GetMemoryRssBytes(),
|
CPUCores: sh.GetCpuCores(), RSSBytes: sh.GetMemoryRssBytes(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
out.CPUTempC, out.CPUPowerW, out.InletTempC = l.GetCpuTempC(), l.GetCpuPowerW(), l.GetInletTempC()
|
||||||
|
for _, g := range l.GetGpus() {
|
||||||
|
out.GPUs = append(out.GPUs, inventory.GPUSensor{
|
||||||
|
Index: int(g.GetIndex()), TempC: g.GetTempC(), MemoryTempC: g.GetMemoryTempC(),
|
||||||
|
PowerW: g.GetPowerW(), PowerLimitW: g.GetPowerLimitW(), Throttle: g.GetThrottle(),
|
||||||
|
})
|
||||||
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -842,3 +867,44 @@ func installable(reg *llmgwv1.Register) []string {
|
|||||||
// The admin API finds the bandwidth test by asserting this; a signature that
|
// The admin API finds the bandwidth test by asserting this; a signature that
|
||||||
// drifted would make it quietly nil.
|
// drifted would make it quietly nil.
|
||||||
var _ fabrictest.Asker = (*Service)(nil)
|
var _ fabrictest.Asker = (*Service)(nil)
|
||||||
|
|
||||||
|
// Upgrade tells a node's agent to replace its binary with a release and
|
||||||
|
// restart, and returns once the agent says the new binary is in place — or
|
||||||
|
// why it is not. The agent then exits and comes back on the new version; the
|
||||||
|
// caller watches for that in the inventory.
|
||||||
|
func (s *Service) Upgrade(ctx context.Context, nodeID, version, releases string) error {
|
||||||
|
s.logMu.Lock()
|
||||||
|
if s.upgradeWaits == nil {
|
||||||
|
s.upgradeWaits = map[string]chan *llmgwv1.UpgradeReply{}
|
||||||
|
}
|
||||||
|
s.logSeq++
|
||||||
|
id := fmt.Sprintf("upgrade-%d", s.logSeq)
|
||||||
|
ch := make(chan *llmgwv1.UpgradeReply, 1)
|
||||||
|
s.upgradeWaits[id] = ch
|
||||||
|
s.logMu.Unlock()
|
||||||
|
defer func() {
|
||||||
|
s.logMu.Lock()
|
||||||
|
delete(s.upgradeWaits, id)
|
||||||
|
s.logMu.Unlock()
|
||||||
|
}()
|
||||||
|
|
||||||
|
if err := s.send(nodeID, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_Upgrade{
|
||||||
|
Upgrade: &llmgwv1.UpgradeAgent{RequestId: id, Version: version, Releases: releases},
|
||||||
|
}}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.log.Info("told a node to upgrade", "node", nodeID, "version", version)
|
||||||
|
// A download of some tens of megabytes on a slow link, and no longer: an
|
||||||
|
// agent too old to know this message never answers.
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
select {
|
||||||
|
case r := <-ch:
|
||||||
|
if r.GetError() != "" {
|
||||||
|
return errors.New(r.GetError())
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return fmt.Errorf("node %s did not answer the upgrade; an agent older than v0.10.4 cannot upgrade itself", nodeID)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net/url"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -74,6 +75,17 @@ type Profile struct {
|
|||||||
// the control plane.
|
// the control plane.
|
||||||
MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"`
|
MinMemoryGB int `json:"min_memory_gb,omitempty" koanf:"min_memory_gb"`
|
||||||
|
|
||||||
|
// CPUCores is the cores a replica of a profile with no GPU is given:
|
||||||
|
// reserved out of its node's CPUs by placement, as memory is, and what
|
||||||
|
// its engine computes on (llama.cpp's --threads).
|
||||||
|
//
|
||||||
|
// Reserved because an engine takes every core it is told about, and two
|
||||||
|
// on one node that each think they have all of them spin against each
|
||||||
|
// other: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second
|
||||||
|
// to under one each. Zero reserves nothing and leaves the thread count to
|
||||||
|
// the node, which is every profile from before this.
|
||||||
|
CPUCores int `json:"cpu_cores,omitempty" koanf:"cpu_cores"`
|
||||||
|
|
||||||
// MinDiskGB is the free space a node needs before it can hold these
|
// MinDiskGB is the free space a node needs before it can hold these
|
||||||
// weights.
|
// weights.
|
||||||
//
|
//
|
||||||
@@ -138,9 +150,30 @@ type Model struct {
|
|||||||
// choice by guesswork.
|
// choice by guesswork.
|
||||||
Description string `json:"description,omitempty" koanf:"description"`
|
Description string `json:"description,omitempty" koanf:"description"`
|
||||||
// Source is the weights path or a HuggingFace id.
|
// Source is the weights path or a HuggingFace id.
|
||||||
Source string `json:"source,omitempty" koanf:"source"`
|
Source string `json:"source,omitempty" koanf:"source"`
|
||||||
Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"`
|
Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"`
|
||||||
Scaling Scaling `json:"scaling,omitempty" koanf:"scaling"`
|
Scaling Scaling `json:"scaling,omitempty" koanf:"scaling"`
|
||||||
|
// Class is how much this model's capacity is worth against another's,
|
||||||
|
// when there is not enough for both: critical, standard or opportunistic.
|
||||||
|
// Empty is standard, or opportunistic for a model loaded on demand. See
|
||||||
|
// EffectiveClass.
|
||||||
|
Class string `json:"class,omitempty" koanf:"class"`
|
||||||
|
// Owner is the part of the tenancy tree whose capacity quota this model
|
||||||
|
// counts against, written org:ID, team:ID or project:ID. Empty is shared:
|
||||||
|
// held by the cluster, counted against nobody.
|
||||||
|
Owner string `json:"owner,omitempty" koanf:"owner"`
|
||||||
|
// External is a model served by someone else — an OpenAI-compatible API
|
||||||
|
// outside the cluster — rather than by replicas placed here. Such a model
|
||||||
|
// has no profiles and nothing is placed for it; the router sends its
|
||||||
|
// requests to the provider. Most useful named in another model's
|
||||||
|
// fallbacks: used only when the cluster has no capacity left. Its
|
||||||
|
// prompts leave the cluster, so which keys may name it is worth an
|
||||||
|
// allowlist.
|
||||||
|
External *External `json:"external,omitempty" koanf:"external"`
|
||||||
|
// Adapters are LoRA adapters served on this model's replicas, each asked
|
||||||
|
// for by its own name as though it were a model. Every replica loads
|
||||||
|
// every adapter, applying one only to a request that names it.
|
||||||
|
Adapters []Adapter `json:"adapters,omitempty" koanf:"adapters"`
|
||||||
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
|
Fallbacks []string `json:"fallbacks,omitempty" koanf:"fallbacks"`
|
||||||
// RoutingProfile selects how the router scores endpoints for this model.
|
// RoutingProfile selects how the router scores endpoints for this model.
|
||||||
RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"`
|
RoutingProfile string `json:"routing_profile,omitempty" koanf:"routing_profile"`
|
||||||
@@ -184,14 +217,17 @@ type Capabilities struct {
|
|||||||
type Catalog struct {
|
type Catalog struct {
|
||||||
models map[string]Model
|
models map[string]Model
|
||||||
aliases map[string]string
|
aliases map[string]string
|
||||||
order []string
|
// adapters maps an adapter's name to the model it runs on.
|
||||||
|
adapters map[string]string
|
||||||
|
order []string
|
||||||
}
|
}
|
||||||
|
|
||||||
// New builds a catalog, rejecting definitions that would fail confusingly later.
|
// New builds a catalog, rejecting definitions that would fail confusingly later.
|
||||||
func New(models []Model) (*Catalog, error) {
|
func New(models []Model) (*Catalog, error) {
|
||||||
c := &Catalog{
|
c := &Catalog{
|
||||||
models: make(map[string]Model, len(models)),
|
models: make(map[string]Model, len(models)),
|
||||||
aliases: map[string]string{},
|
aliases: map[string]string{},
|
||||||
|
adapters: map[string]string{},
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := range models {
|
for i := range models {
|
||||||
@@ -220,6 +256,18 @@ func New(models []Model) (*Catalog, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Adapters are named like models, so they share the namespace too.
|
||||||
|
for _, name := range c.order {
|
||||||
|
for _, a := range c.models[name].Adapters {
|
||||||
|
_, isModel := c.models[a.Name]
|
||||||
|
_, isAlias := c.aliases[a.Name]
|
||||||
|
if _, isAdapter := c.adapters[a.Name]; isModel || isAlias || isAdapter {
|
||||||
|
return nil, fmt.Errorf("model %q: adapter %q collides with a model, an alias or another adapter", name, a.Name)
|
||||||
|
}
|
||||||
|
c.adapters[a.Name] = name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Fallbacks are resolved now rather than at request time: a typo in a
|
// Fallbacks are resolved now rather than at request time: a typo in a
|
||||||
// fallback should stop a deployment, not surface during an outage when the
|
// fallback should stop a deployment, not surface during an outage when the
|
||||||
// fallback is finally needed.
|
// fallback is finally needed.
|
||||||
@@ -285,9 +333,22 @@ func (m Model) validate() error {
|
|||||||
if m.Source == "" {
|
if m.Source == "" {
|
||||||
return fmt.Errorf("model %q: source is required", m.Name)
|
return fmt.Errorf("model %q: source is required", m.Name)
|
||||||
}
|
}
|
||||||
if len(m.Profiles) == 0 {
|
if err := validateAdapters(m); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if m.External != nil {
|
||||||
|
if err := m.External.validate(m.Name, len(m.Profiles)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else if len(m.Profiles) == 0 {
|
||||||
return fmt.Errorf("model %q: at least one profile is required", m.Name)
|
return fmt.Errorf("model %q: at least one profile is required", m.Name)
|
||||||
}
|
}
|
||||||
|
if m.Owner != "" && !validOwner(m.Owner) {
|
||||||
|
return fmt.Errorf("model %q: owner %q is not org:ID, team:ID or project:ID", m.Name, m.Owner)
|
||||||
|
}
|
||||||
|
if !validClass(m.Class) {
|
||||||
|
return fmt.Errorf("model %q: class %q is not one of critical, standard, opportunistic", m.Name, m.Class)
|
||||||
|
}
|
||||||
if m.Scaling.Max < m.Scaling.Min {
|
if m.Scaling.Max < m.Scaling.Min {
|
||||||
return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min)
|
return fmt.Errorf("model %q: scaling max %d is below min %d", m.Name, m.Scaling.Max, m.Scaling.Min)
|
||||||
}
|
}
|
||||||
@@ -302,6 +363,13 @@ func (m Model) validate() error {
|
|||||||
}
|
}
|
||||||
seen[p.Name] = struct{}{}
|
seen[p.Name] = struct{}{}
|
||||||
|
|
||||||
|
for k := range p.Env {
|
||||||
|
if bad, why := engines.UnsafeEnv(k); bad {
|
||||||
|
return fmt.Errorf("model %q profile %q: env %s is not allowed: %s",
|
||||||
|
m.Name, p.Name, k, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if p.GPUClass == "" && !cpuCapable(p.Engine) {
|
if p.GPUClass == "" && !cpuCapable(p.Engine) {
|
||||||
return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q",
|
return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q",
|
||||||
m.Name, p.Name, p.Engine)
|
m.Name, p.Name, p.Engine)
|
||||||
@@ -324,6 +392,23 @@ func (m Model) validate() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Resolve maps a client-supplied name or alias onto a model.
|
// Resolve maps a client-supplied name or alias onto a model.
|
||||||
|
// ResolveAdapter finds an adapter by its name: the model it runs on, the
|
||||||
|
// adapter, and its index among the model's adapters — which is the id the
|
||||||
|
// engine knows it by.
|
||||||
|
func (c *Catalog) ResolveAdapter(name string) (Model, Adapter, int, bool) {
|
||||||
|
base, ok := c.adapters[name]
|
||||||
|
if !ok {
|
||||||
|
return Model{}, Adapter{}, 0, false
|
||||||
|
}
|
||||||
|
m := c.models[base]
|
||||||
|
for i, a := range m.Adapters {
|
||||||
|
if a.Name == name {
|
||||||
|
return m, a, i, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Model{}, Adapter{}, 0, false
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Catalog) Resolve(name string) (Model, bool) {
|
func (c *Catalog) Resolve(name string) (Model, bool) {
|
||||||
if m, ok := c.models[name]; ok {
|
if m, ok := c.models[name]; ok {
|
||||||
return m, true
|
return m, true
|
||||||
@@ -373,6 +458,7 @@ func (m Model) PlacementProfiles() []placement.Profile {
|
|||||||
NodeSelector: p.NodeSelector,
|
NodeSelector: p.NodeSelector,
|
||||||
MinMemoryGB: p.MinMemoryGB,
|
MinMemoryGB: p.MinMemoryGB,
|
||||||
MinDiskGB: p.MinDiskGB,
|
MinDiskGB: p.MinDiskGB,
|
||||||
|
CPUCores: p.CPUCores,
|
||||||
// The model's source, so placement can prefer nodes that already
|
// The model's source, so placement can prefer nodes that already
|
||||||
// hold the weights rather than one that must first move them.
|
// hold the weights rather than one that must first move them.
|
||||||
Source: m.Source,
|
Source: m.Source,
|
||||||
@@ -399,6 +485,7 @@ func (m Model) EngineProfile(name string) (engines.Profile, bool) {
|
|||||||
Env: p.Env,
|
Env: p.Env,
|
||||||
ModelPath: m.Source,
|
ModelPath: m.Source,
|
||||||
ServedModelName: m.Name,
|
ServedModelName: m.Name,
|
||||||
|
Adapters: m.engineAdapters(),
|
||||||
}, true
|
}, true
|
||||||
}
|
}
|
||||||
return engines.Profile{}, false
|
return engines.Profile{}, false
|
||||||
@@ -468,3 +555,152 @@ func (p Profile) Fingerprint() string {
|
|||||||
sum := sha256.Sum256(b)
|
sum := sha256.Sum256(b)
|
||||||
return hex.EncodeToString(sum[:8])
|
return hex.EncodeToString(sum[:8])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SuggestCores is the cores to give a replica with no GPU on nodes of these
|
||||||
|
// CPU counts: half of the smallest, so two replicas share any of them without
|
||||||
|
// competing. Zero when no node says, which reserves nothing.
|
||||||
|
func SuggestCores(cpuCounts []int) int {
|
||||||
|
smallest := 0
|
||||||
|
for _, c := range cpuCounts {
|
||||||
|
if c > 0 && (smallest == 0 || c < smallest) {
|
||||||
|
smallest = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if smallest == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return max(smallest/2, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The capacity classes, lowest first.
|
||||||
|
const (
|
||||||
|
ClassOpportunistic = "opportunistic"
|
||||||
|
ClassStandard = "standard"
|
||||||
|
ClassCritical = "critical"
|
||||||
|
)
|
||||||
|
|
||||||
|
func validClass(c string) bool {
|
||||||
|
return c == "" || c == ClassOpportunistic || c == ClassStandard || c == ClassCritical
|
||||||
|
}
|
||||||
|
|
||||||
|
// EffectiveClass is the class a model is treated as: what it says, or, when
|
||||||
|
// it says nothing, opportunistic for one loaded on demand — it holds capacity
|
||||||
|
// only while asked for, and has always been the first to give it up — and
|
||||||
|
// standard for everything else.
|
||||||
|
func (m Model) EffectiveClass() string {
|
||||||
|
if m.Class != "" {
|
||||||
|
return m.Class
|
||||||
|
}
|
||||||
|
if m.Scaling.OnDemand() {
|
||||||
|
return ClassOpportunistic
|
||||||
|
}
|
||||||
|
return ClassStandard
|
||||||
|
}
|
||||||
|
|
||||||
|
// ClassRank orders classes: a higher rank may take capacity from a lower one.
|
||||||
|
func ClassRank(c string) int {
|
||||||
|
switch c {
|
||||||
|
case ClassCritical:
|
||||||
|
return 3
|
||||||
|
case ClassStandard, "":
|
||||||
|
return 2
|
||||||
|
default:
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func validOwner(o string) bool {
|
||||||
|
for _, p := range []string{"org:", "team:", "project:"} {
|
||||||
|
if strings.HasPrefix(o, p) && len(o) > len(p) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// External is a provider outside the cluster that serves a model.
|
||||||
|
type External struct {
|
||||||
|
// URL is the provider's base address, without /v1: https://api.openai.com.
|
||||||
|
URL string `json:"url" koanf:"url"`
|
||||||
|
// Model is what the provider calls it; requests are rewritten to it.
|
||||||
|
Model string `json:"model" koanf:"model"`
|
||||||
|
// APIKeyFile holds the provider's key, on the control plane. A file
|
||||||
|
// rather than the key itself, so the key is in neither the catalog, the
|
||||||
|
// store nor the audit log; empty for a provider that asks for none.
|
||||||
|
APIKeyFile string `json:"api_key_file,omitempty" koanf:"api_key_file"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *External) validate(model string, profiles int) error {
|
||||||
|
if profiles > 0 {
|
||||||
|
return fmt.Errorf("model %q: an external model is served elsewhere and has no profiles", model)
|
||||||
|
}
|
||||||
|
u, err := url.Parse(e.URL)
|
||||||
|
if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" {
|
||||||
|
return fmt.Errorf("model %q: external url %q is not an http(s) address", model, e.URL)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(e.Model) == "" {
|
||||||
|
return fmt.Errorf("model %q: external model needs the name the provider knows it by", model)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsExternal reports whether a model is served outside the cluster.
|
||||||
|
func (m Model) IsExternal() bool { return m.External != nil }
|
||||||
|
|
||||||
|
// Adapter is a LoRA adapter served on a model's replicas.
|
||||||
|
type Adapter struct {
|
||||||
|
// Name is what clients ask for, like a model name.
|
||||||
|
Name string `json:"name" koanf:"name"`
|
||||||
|
// Source is the adapter file (GGUF for llama.cpp), fetched by each node
|
||||||
|
// as the model's weights are.
|
||||||
|
Source string `json:"source" koanf:"source"`
|
||||||
|
// Scale is how strongly it is applied; zero takes 1.
|
||||||
|
Scale float64 `json:"scale,omitempty" koanf:"scale"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// EffectiveScale is the scale an adapter is applied at.
|
||||||
|
func (a Adapter) EffectiveScale() float64 {
|
||||||
|
if a.Scale <= 0 {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return a.Scale
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateAdapters(m Model) error {
|
||||||
|
if len(m.Adapters) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if m.External != nil {
|
||||||
|
return fmt.Errorf("model %q: an external model cannot carry adapters", m.Name)
|
||||||
|
}
|
||||||
|
for _, p := range m.Profiles {
|
||||||
|
// llama.cpp loads adapters at start and applies them per request;
|
||||||
|
// the other engines are not driven that way yet.
|
||||||
|
if p.Engine != "llamacpp" && p.Engine != "sim" {
|
||||||
|
return fmt.Errorf("model %q profile %q: adapters are served by llamacpp profiles only", m.Name, p.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, a := range m.Adapters {
|
||||||
|
switch {
|
||||||
|
case a.Name == "" || a.Source == "":
|
||||||
|
return fmt.Errorf("model %q: an adapter needs a name and a source", m.Name)
|
||||||
|
case a.Name == m.Name:
|
||||||
|
return fmt.Errorf("model %q: an adapter cannot have the model's own name", m.Name)
|
||||||
|
case seen[a.Name]:
|
||||||
|
return fmt.Errorf("model %q: adapter %q is declared twice", m.Name, a.Name)
|
||||||
|
case a.Scale < 0:
|
||||||
|
return fmt.Errorf("model %q: adapter %q has a negative scale", m.Name, a.Name)
|
||||||
|
}
|
||||||
|
seen[a.Name] = true
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m Model) engineAdapters() []engines.Adapter {
|
||||||
|
var out []engines.Adapter
|
||||||
|
for _, a := range m.Adapters {
|
||||||
|
out = append(out, engines.Adapter{Name: a.Name, Path: a.Source})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -302,3 +302,22 @@ func TestSimulatedNeedsEveryProfileToBeSimulated(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAProfileCannotSetWhatRunsCodeInTheEngine(t *testing.T) {
|
||||||
|
// A profile's env reaches an engine started as root. LD_PRELOAD from a
|
||||||
|
// model definition would make "add a model" into "run this everywhere".
|
||||||
|
for _, k := range []string{"LD_PRELOAD", "ld_library_path", "PYTHONSTARTUP", "PATH", "BASH_ENV"} {
|
||||||
|
m := Model{Name: "m", Source: "http://x/m.gguf", Profiles: []Profile{{
|
||||||
|
Name: "cpu", Engine: "llamacpp", Env: map[string]string{k: "/tmp/x"},
|
||||||
|
}}}
|
||||||
|
if _, err := New([]Model{m}); err == nil || !strings.Contains(err.Error(), "not allowed") {
|
||||||
|
t.Errorf("%s: %v", k, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ok := Model{Name: "m", Source: "http://x/m.gguf", Profiles: []Profile{{
|
||||||
|
Name: "cpu", Engine: "llamacpp", Env: map[string]string{"OMP_NUM_THREADS": "4", "PYTHONUNBUFFERED": "1"},
|
||||||
|
}}}
|
||||||
|
if _, err := New([]Model{ok}); err != nil {
|
||||||
|
t.Errorf("ordinary tuning refused: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,8 +4,12 @@ package control
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -131,6 +135,16 @@ func (c *Controller) collectUsage() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
c.rec.RecordUsage(c.usage.LastUsed())
|
c.rec.RecordUsage(c.usage.LastUsed())
|
||||||
|
// And what the replicas are doing, when the source can say: what adds
|
||||||
|
// and removes replicas between a model's min and max.
|
||||||
|
if l, ok := c.usage.(LoadSource); ok {
|
||||||
|
c.rec.RecordLoad(l.ModelLoad())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadSource reports each model's replicas' load, keyed by catalog name.
|
||||||
|
type LoadSource interface {
|
||||||
|
ModelLoad() map[string]reconciler.Load
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReplaceCatalog puts a new catalog in force, without a restart.
|
// ReplaceCatalog puts a new catalog in force, without a restart.
|
||||||
@@ -168,6 +182,9 @@ func (c *Controller) Nodes() []inventory.Node { return c.inv.Nodes() }
|
|||||||
func (c *Controller) FreeGPUs(nodeID string) []int { return c.inv.FreeGPUs(nodeID) }
|
func (c *Controller) FreeGPUs(nodeID string) []int { return c.inv.FreeGPUs(nodeID) }
|
||||||
func (c *Controller) Replicas() []reconciler.Replica { return c.rec.Replicas() }
|
func (c *Controller) Replicas() []reconciler.Replica { return c.rec.Replicas() }
|
||||||
|
|
||||||
|
// ModelHealth is how each model in the catalog stands.
|
||||||
|
func (c *Controller) ModelHealth() []reconciler.ModelHealth { return c.rec.ModelHealth() }
|
||||||
|
|
||||||
// Reconciler exposes the reconciliation loop, for status reports and ctl.
|
// Reconciler exposes the reconciliation loop, for status reports and ctl.
|
||||||
func (c *Controller) Reconciler() *reconciler.Reconciler { return c.rec }
|
func (c *Controller) Reconciler() *reconciler.Reconciler { return c.rec }
|
||||||
|
|
||||||
@@ -205,6 +222,17 @@ func (c *Controller) WeightRemover() WeightRemover {
|
|||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Upgrader can tell a node's agent to upgrade itself.
|
||||||
|
type Upgrader interface {
|
||||||
|
Upgrade(ctx context.Context, nodeID, version, releases string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upgrader returns the dispatcher's upgrade capability, or nil.
|
||||||
|
func (c *Controller) Upgrader() Upgrader {
|
||||||
|
u, _ := c.disp.(Upgrader)
|
||||||
|
return u
|
||||||
|
}
|
||||||
|
|
||||||
// LogReader can ask a node for the recent lines of one of its logs.
|
// LogReader can ask a node for the recent lines of one of its logs.
|
||||||
type LogReader interface {
|
type LogReader interface {
|
||||||
Logs(ctx context.Context, nodeID, source string, lines int) (out, sources []string, err error)
|
Logs(ctx context.Context, nodeID, source string, lines int) (out, sources []string, err error)
|
||||||
@@ -304,12 +332,41 @@ func (c *Controller) endpoints() []endpoint.Info {
|
|||||||
ID: rep.ID,
|
ID: rep.ID,
|
||||||
URL: strings.TrimRight(rep.Endpoint, "/"),
|
URL: strings.TrimRight(rep.Endpoint, "/"),
|
||||||
Models: names,
|
Models: names,
|
||||||
|
APIKey: c.engineKey(rep.ID),
|
||||||
Engine: prof.Engine,
|
Engine: prof.Engine,
|
||||||
Pool: rep.Pool,
|
Pool: rep.Pool,
|
||||||
Weight: rep.Weight,
|
Weight: rep.Weight,
|
||||||
Priority: prof.Priority,
|
Priority: prof.Priority,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
return append(out, c.externalEndpoints()...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// externalEndpoints are the models served outside the cluster, as endpoints:
|
||||||
|
// one each, with the provider's key read from its file every time, so a key
|
||||||
|
// rotated on disk is used from the next publish on.
|
||||||
|
func (c *Controller) externalEndpoints() []endpoint.Info {
|
||||||
|
var out []endpoint.Info
|
||||||
|
for _, m := range c.cat.Load().Models() {
|
||||||
|
if !m.IsExternal() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := ""
|
||||||
|
if f := m.External.APIKeyFile; f != "" {
|
||||||
|
b, err := os.ReadFile(f) //nolint:gosec // the operator's own path
|
||||||
|
if err != nil {
|
||||||
|
c.log.Warn("an external model's key could not be read; not offered", "model", m.Name, "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key = strings.TrimSpace(string(b))
|
||||||
|
}
|
||||||
|
out = append(out, endpoint.Info{
|
||||||
|
ID: "external:" + m.Name, URL: strings.TrimRight(m.External.URL, "/"),
|
||||||
|
Models: append([]string{m.Name}, m.Aliases...), APIKey: key,
|
||||||
|
Engine: endpoint.EngineExternal, Pool: "external", Weight: 1,
|
||||||
|
UpstreamModel: m.External.Model,
|
||||||
|
})
|
||||||
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -317,7 +374,37 @@ func (c *Controller) endpoints() []endpoint.Info {
|
|||||||
func fingerprint(infos []endpoint.Info) string {
|
func fingerprint(infos []endpoint.Info) string {
|
||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
for _, i := range infos {
|
for _, i := range infos {
|
||||||
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority)
|
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d|%s|%x;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority,
|
||||||
|
i.UpstreamModel, sha256.Sum256([]byte(i.APIKey)))
|
||||||
}
|
}
|
||||||
return b.String()
|
return b.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EngineKey is the key a replica's engine demands, empty when engines run
|
||||||
|
// without one.
|
||||||
|
func (c *Controller) EngineKey(replicaID string) string { return c.engineKey(replicaID) }
|
||||||
|
|
||||||
|
func (c *Controller) engineKey(replicaID string) string {
|
||||||
|
if c.cfg.Reconciler.EngineKey == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return c.cfg.Reconciler.EngineKey(replicaID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// EngineKeys derives each replica's engine key from one secret.
|
||||||
|
//
|
||||||
|
// HMAC rather than a stored random key per replica: the router needs the key
|
||||||
|
// for every replica it sends to, including ones adopted after a restart that
|
||||||
|
// this process never started, and a second control plane has to agree. A
|
||||||
|
// function of a shared secret and the replica id gives all three without a
|
||||||
|
// table to keep in step. A key taken from one node opens that replica only.
|
||||||
|
func EngineKeys(secret []byte) func(replicaID string) string {
|
||||||
|
return func(replicaID string) string {
|
||||||
|
m := hmac.New(sha256.New, secret)
|
||||||
|
m.Write([]byte("llmgw engine key\x00" + replicaID))
|
||||||
|
return "llmgw-" + hex.EncodeToString(m.Sum(nil))[:40]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// QuotaUse is every capacity quota beside what its scope holds.
|
||||||
|
func (c *Controller) QuotaUse() []reconciler.QuotaUse { return c.rec.QuotaUse() }
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
package control
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestEngineKeysAreTheSameForTheSameReplicaAndNoOther(t *testing.T) {
|
||||||
|
// A control plane that restarts, or a second one, must compute the key a
|
||||||
|
// running engine was started with; a key taken from one node must open
|
||||||
|
// that replica only.
|
||||||
|
a, b := EngineKeys([]byte("s")), EngineKeys([]byte("s"))
|
||||||
|
if a("r1") != b("r1") {
|
||||||
|
t.Error("the same secret gave two keys for one replica")
|
||||||
|
}
|
||||||
|
if a("r1") == a("r2") {
|
||||||
|
t.Error("two replicas share a key")
|
||||||
|
}
|
||||||
|
if a("r1") == EngineKeys([]byte("other"))("r1") {
|
||||||
|
t.Error("the key does not depend on the secret")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
package control
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||||
|
)
|
||||||
|
|
||||||
|
type recordSink struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
infos []endpoint.Info
|
||||||
|
times int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *recordSink) SetEndpoints(_ context.Context, infos []endpoint.Info) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.infos, s.times = infos, s.times+1
|
||||||
|
}
|
||||||
|
|
||||||
|
type noDispatch struct{}
|
||||||
|
|
||||||
|
func (noDispatch) StartShard(context.Context, reconciler.ShardCommand) error { return nil }
|
||||||
|
func (noDispatch) StopShard(context.Context, string, string) error { return nil }
|
||||||
|
func (noDispatch) DrainShard(context.Context, string, string) error { return nil }
|
||||||
|
|
||||||
|
func TestAnExternalModelIsPublishedWithItsKeyAndNothingIsPlacedForIt(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
keyFile := filepath.Join(dir, "provider.key")
|
||||||
|
if err := os.WriteFile(keyFile, []byte("sk-provider-1\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
models := []catalog.Model{
|
||||||
|
{Name: "cloud", Source: "external", Aliases: []string{"gpt"}, Scaling: catalog.Scaling{Min: 3, Max: 3},
|
||||||
|
External: &catalog.External{URL: "https://api.example.com/", Model: "gpt-4o-mini", APIKeyFile: keyFile}},
|
||||||
|
{Name: "gone", Source: "external",
|
||||||
|
External: &catalog.External{URL: "https://api.example.com", Model: "m", APIKeyFile: filepath.Join(dir, "missing")}},
|
||||||
|
}
|
||||||
|
cat, err := catalog.New(models)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sink := &recordSink{}
|
||||||
|
c := New(DefaultConfig(), catalog.NewLive(cat), noDispatch{}, sink, slog.New(slog.DiscardHandler))
|
||||||
|
c.Publish(t.Context())
|
||||||
|
|
||||||
|
if len(sink.infos) != 1 {
|
||||||
|
t.Fatalf("published %+v; the model whose key is missing must not be offered", sink.infos)
|
||||||
|
}
|
||||||
|
e := sink.infos[0]
|
||||||
|
if e.ID != "external:cloud" || e.URL != "https://api.example.com" || e.APIKey != "sk-provider-1" ||
|
||||||
|
e.UpstreamModel != "gpt-4o-mini" || e.Engine != endpoint.EngineExternal || len(e.Models) != 2 {
|
||||||
|
t.Errorf("endpoint %+v", e)
|
||||||
|
}
|
||||||
|
// A key rotated on disk is published on the next pass.
|
||||||
|
_ = os.WriteFile(keyFile, []byte("sk-provider-2"), 0o600)
|
||||||
|
c.Publish(t.Context())
|
||||||
|
if sink.times != 2 || sink.infos[0].APIKey != "sk-provider-2" {
|
||||||
|
t.Errorf("the rotated key was not published: %d pushes, key %q", sink.times, sink.infos[0].APIKey)
|
||||||
|
}
|
||||||
|
// Scaling of 3 or not, nothing is wanted of the cluster for a model
|
||||||
|
// served elsewhere, and so it never reads as short or unplaceable.
|
||||||
|
for _, h := range c.Reconciler().ModelHealth() {
|
||||||
|
if h.Model == "cloud" && (h.Desired != 0 || h.Unplaceable) {
|
||||||
|
t.Errorf("an external model wants replicas: %+v", h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -39,9 +39,29 @@ type Source interface {
|
|||||||
// derived from state that already exists, so computing them when asked is both
|
// derived from state that already exists, so computing them when asked is both
|
||||||
// simpler and fresher than maintaining a parallel copy that can drift. Nothing
|
// simpler and fresher than maintaining a parallel copy that can drift. Nothing
|
||||||
// runs between scrapes.
|
// runs between scrapes.
|
||||||
|
// HealthSource adds what the reconciler knows about each model and where
|
||||||
|
// engine installs stand, and whether this control plane is the one placing
|
||||||
|
// work. Asked for by type assertion, so a source without it — a test's —
|
||||||
|
// publishes what it did before.
|
||||||
|
type HealthSource interface {
|
||||||
|
ModelHealth() []reconciler.ModelHealth
|
||||||
|
IsLeader() bool
|
||||||
|
}
|
||||||
|
|
||||||
type Collector struct {
|
type Collector struct {
|
||||||
src Source
|
src Source
|
||||||
|
|
||||||
|
modelDesired *prometheus.Desc
|
||||||
|
modelFailures *prometheus.Desc
|
||||||
|
modelParked *prometheus.Desc
|
||||||
|
modelUnplaceable *prometheus.Desc
|
||||||
|
modelPreempted *prometheus.Desc
|
||||||
|
capacityHolds *prometheus.Desc
|
||||||
|
capacityGuarantee *prometheus.Desc
|
||||||
|
capacityLimit *prometheus.Desc
|
||||||
|
engineInstall *prometheus.Desc
|
||||||
|
leader *prometheus.Desc
|
||||||
|
|
||||||
nodeUp *prometheus.Desc
|
nodeUp *prometheus.Desc
|
||||||
nodeGPUs *prometheus.Desc
|
nodeGPUs *prometheus.Desc
|
||||||
nodeGPUsFree *prometheus.Desc
|
nodeGPUsFree *prometheus.Desc
|
||||||
@@ -54,6 +74,13 @@ type Collector struct {
|
|||||||
gpuMemoryUsed *prometheus.Desc
|
gpuMemoryUsed *prometheus.Desc
|
||||||
gpuUtil *prometheus.Desc
|
gpuUtil *prometheus.Desc
|
||||||
gpuTemp *prometheus.Desc
|
gpuTemp *prometheus.Desc
|
||||||
|
gpuMemTemp *prometheus.Desc
|
||||||
|
gpuPower *prometheus.Desc
|
||||||
|
gpuPowerLimit *prometheus.Desc
|
||||||
|
gpuThrottled *prometheus.Desc
|
||||||
|
nodeCPUTemp *prometheus.Desc
|
||||||
|
nodeCPUPower *prometheus.Desc
|
||||||
|
nodeInletTemp *prometheus.Desc
|
||||||
gpuHealthy *prometheus.Desc
|
gpuHealthy *prometheus.Desc
|
||||||
gpuNVLink *prometheus.Desc
|
gpuNVLink *prometheus.Desc
|
||||||
gpuECC *prometheus.Desc
|
gpuECC *prometheus.Desc
|
||||||
@@ -122,6 +149,21 @@ func New(src Source) *Collector {
|
|||||||
"How busy a GPU was over the last sample period. Absent when the driver does not "+
|
"How busy a GPU was over the last sample period. Absent when the driver does not "+
|
||||||
"report it, which is not the same as idle. Memory used says a model is loaded; "+
|
"report it, which is not the same as idle. Memory used says a model is loaded; "+
|
||||||
"this says whether it is doing anything.", gpu, nil),
|
"this says whether it is doing anything.", gpu, nil),
|
||||||
|
gpuMemTemp: prometheus.NewDesc("llmgw_gpu_memory_temperature_celsius",
|
||||||
|
"A GPU's memory (HBM) temperature, where the driver reports it.", gpu, nil),
|
||||||
|
gpuPower: prometheus.NewDesc("llmgw_gpu_power_watts",
|
||||||
|
"A GPU's power draw now.", gpu, nil),
|
||||||
|
gpuPowerLimit: prometheus.NewDesc("llmgw_gpu_power_limit_watts",
|
||||||
|
"The power limit a GPU is held to.", gpu, nil),
|
||||||
|
gpuThrottled: prometheus.NewDesc("llmgw_gpu_throttled",
|
||||||
|
"1 while something holds a GPU's clocks down: sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake.",
|
||||||
|
append(gpu, "reason"), nil),
|
||||||
|
nodeCPUTemp: prometheus.NewDesc("llmgw_node_cpu_temperature_celsius",
|
||||||
|
"A node's hottest CPU sensor.", node, nil),
|
||||||
|
nodeCPUPower: prometheus.NewDesc("llmgw_node_cpu_power_watts",
|
||||||
|
"A node's CPU packages' power draw, from RAPL.", node, nil),
|
||||||
|
nodeInletTemp: prometheus.NewDesc("llmgw_node_inlet_temperature_celsius",
|
||||||
|
"A node's inlet or ambient air sensor, where its board has one.", node, nil),
|
||||||
gpuTemp: prometheus.NewDesc("llmgw_gpu_temperature_celsius",
|
gpuTemp: prometheus.NewDesc("llmgw_gpu_temperature_celsius",
|
||||||
"A GPU's reported temperature.", gpu, nil),
|
"A GPU's reported temperature.", gpu, nil),
|
||||||
gpuHealthy: prometheus.NewDesc("llmgw_gpu_healthy",
|
gpuHealthy: prometheus.NewDesc("llmgw_gpu_healthy",
|
||||||
@@ -137,7 +179,38 @@ func New(src Source) *Collector {
|
|||||||
replicaState: prometheus.NewDesc("llmgw_replicas",
|
replicaState: prometheus.NewDesc("llmgw_replicas",
|
||||||
"Replicas by model and state.", []string{"model", "state"}, nil),
|
"Replicas by model and state.", []string{"model", "state"}, nil),
|
||||||
replicaReady: prometheus.NewDesc("llmgw_model_replicas_ready",
|
replicaReady: prometheus.NewDesc("llmgw_model_replicas_ready",
|
||||||
"Replicas of a model that are serving.", []string{"model"}, nil),
|
"Replicas of a model that are serving. Published, as 0 when none are, for every "+
|
||||||
|
"model that should be running; a stopped model, or an on-demand one nobody is "+
|
||||||
|
"asking for, publishes nothing.", []string{"model"}, nil),
|
||||||
|
modelDesired: prometheus.NewDesc("llmgw_model_replicas_desired",
|
||||||
|
"Replicas a model should have now: its minimum, or for an on-demand model 1 while "+
|
||||||
|
"it is being asked for.", []string{"model"}, nil),
|
||||||
|
modelFailures: prometheus.NewDesc("llmgw_model_start_failures",
|
||||||
|
"Replicas of a model that failed to start in a row. Resets when one comes up.",
|
||||||
|
[]string{"model"}, nil),
|
||||||
|
modelParked: prometheus.NewDesc("llmgw_model_parked",
|
||||||
|
"1 when a model failed so often in a row that it is no longer retried.",
|
||||||
|
[]string{"model"}, nil),
|
||||||
|
modelUnplaceable: prometheus.NewDesc("llmgw_model_unplaceable",
|
||||||
|
"1 when a model is short of replicas and the last attempt found no node to place one on.",
|
||||||
|
[]string{"model"}, nil),
|
||||||
|
capacityHolds: prometheus.NewDesc("llmgw_capacity_holds",
|
||||||
|
"What a part of the tenancy tree holds of a resource, its own models' replicas and those beneath it.",
|
||||||
|
[]string{"scope", "resource"}, nil),
|
||||||
|
capacityGuarantee: prometheus.NewDesc("llmgw_capacity_guarantee",
|
||||||
|
"What a part of the tenancy tree is guaranteed of a resource.",
|
||||||
|
[]string{"scope", "resource"}, nil),
|
||||||
|
capacityLimit: prometheus.NewDesc("llmgw_capacity_limit",
|
||||||
|
"The most of a resource a part of the tenancy tree may hold.",
|
||||||
|
[]string{"scope", "resource"}, nil),
|
||||||
|
modelPreempted: prometheus.NewDesc("llmgw_model_preempted_total",
|
||||||
|
"Replicas of a model stopped to make room for a model of a higher capacity class.",
|
||||||
|
[]string{"model", "class"}, nil),
|
||||||
|
engineInstall: prometheus.NewDesc("llmgw_node_engine_install",
|
||||||
|
"Where installing an engine stands on a node: 1 for the current state.",
|
||||||
|
[]string{"node", "engine", "version", "state"}, nil),
|
||||||
|
leader: prometheus.NewDesc("llmgw_control_plane_leader",
|
||||||
|
"1 when this control plane is the one placing work, 0 when it is a standby.", nil, nil),
|
||||||
replicaIdle: prometheus.NewDesc("llmgw_replica_idle_seconds",
|
replicaIdle: prometheus.NewDesc("llmgw_replica_idle_seconds",
|
||||||
"Seconds since a replica was last sent a request. Absent when nothing has been reported, "+
|
"Seconds since a replica was last sent a request. Absent when nothing has been reported, "+
|
||||||
"which is not the same as idle.", []string{"model", "replica"}, nil),
|
"which is not the same as idle.", []string{"model", "replica"}, nil),
|
||||||
@@ -199,8 +272,12 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
|
|||||||
c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
|
c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
|
||||||
c.gpuNVLJoined, c.imexUp,
|
c.gpuNVLJoined, c.imexUp,
|
||||||
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
|
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
|
||||||
|
c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp,
|
||||||
c.gpuNVLink, c.gpuECC,
|
c.gpuNVLink, c.gpuECC,
|
||||||
c.replicaState, c.replicaReady, c.replicaIdle,
|
c.replicaState, c.replicaReady, c.replicaIdle,
|
||||||
|
c.modelDesired, c.modelFailures, c.modelParked, c.modelUnplaceable, c.modelPreempted,
|
||||||
|
c.capacityHolds, c.capacityGuarantee, c.capacityLimit,
|
||||||
|
c.engineInstall, c.leader,
|
||||||
} {
|
} {
|
||||||
ch <- d
|
ch <- d
|
||||||
}
|
}
|
||||||
@@ -209,6 +286,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
|
|||||||
// Collect implements prometheus.Collector.
|
// Collect implements prometheus.Collector.
|
||||||
func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
|
c.collectQuotas(ch)
|
||||||
|
|
||||||
loads := map[string]*inventory.Load{}
|
loads := map[string]*inventory.Load{}
|
||||||
for _, n := range c.src.Nodes() {
|
for _, n := range c.src.Nodes() {
|
||||||
@@ -242,6 +320,16 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if l := n.Load; l != nil && now.Sub(l.At) < loadFresh {
|
if l := n.Load; l != nil && now.Sub(l.At) < loadFresh {
|
||||||
|
// Each only when measured: a VM has none of these, and a zero
|
||||||
|
// would read as a machine at 0 °C drawing nothing.
|
||||||
|
for _, v := range []struct {
|
||||||
|
d *prometheus.Desc
|
||||||
|
val float64
|
||||||
|
}{{c.nodeCPUTemp, l.CPUTempC}, {c.nodeCPUPower, l.CPUPowerW}, {c.nodeInletTemp, l.InletTempC}} {
|
||||||
|
if v.val > 0 {
|
||||||
|
ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, n.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
ch <- prometheus.MustNewConstMetric(c.nodeCPU, prometheus.GaugeValue, l.CPUPercent, n.ID)
|
ch <- prometheus.MustNewConstMetric(c.nodeCPU, prometheus.GaugeValue, l.CPUPercent, n.ID)
|
||||||
if l.CPUCount > 0 {
|
if l.CPUCount > 0 {
|
||||||
ch <- prometheus.MustNewConstMetric(c.nodeCPUs, prometheus.GaugeValue, float64(l.CPUCount), n.ID)
|
ch <- prometheus.MustNewConstMetric(c.nodeCPUs, prometheus.GaugeValue, float64(l.CPUCount), n.ID)
|
||||||
@@ -311,7 +399,11 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
|||||||
ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue,
|
ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue,
|
||||||
float64(g.UtilizationPercent), labels...)
|
float64(g.UtilizationPercent), labels...)
|
||||||
}
|
}
|
||||||
if g.Health.TemperatureC > 0 {
|
// The sensors' fresh reading when there is one; the temperature
|
||||||
|
// read at registration otherwise, which is all an older agent sends.
|
||||||
|
if s, ok := sensorFor(n.Load, g.Index, now); ok {
|
||||||
|
c.gpuSensor(ch, s, labels)
|
||||||
|
} else if g.Health.TemperatureC > 0 {
|
||||||
ch <- prometheus.MustNewConstMetric(c.gpuTemp, prometheus.GaugeValue,
|
ch <- prometheus.MustNewConstMetric(c.gpuTemp, prometheus.GaugeValue,
|
||||||
float64(g.Health.TemperatureC), labels...)
|
float64(g.Health.TemperatureC), labels...)
|
||||||
}
|
}
|
||||||
@@ -387,8 +479,106 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
|||||||
ch <- prometheus.MustNewConstMetric(c.replicaState, prometheus.GaugeValue,
|
ch <- prometheus.MustNewConstMetric(c.replicaState, prometheus.GaugeValue,
|
||||||
float64(v), k[0], k[1])
|
float64(v), k[0], k[1])
|
||||||
}
|
}
|
||||||
for model, v := range ready {
|
hs, ok := c.src.(HealthSource)
|
||||||
ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue,
|
if !ok {
|
||||||
float64(v), model)
|
for model, v := range ready {
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue,
|
||||||
|
float64(v), model)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, n := range c.src.Nodes() {
|
||||||
|
for _, p := range n.Provision {
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.engineInstall, prometheus.GaugeValue, 1,
|
||||||
|
n.ID, p.Engine, p.Version, p.State)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lead := 0.0
|
||||||
|
if hs.IsLeader() {
|
||||||
|
lead = 1
|
||||||
|
}
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.leader, prometheus.GaugeValue, lead)
|
||||||
|
for _, h := range hs.ModelHealth() {
|
||||||
|
// Ready is published at zero for what should be running — the one
|
||||||
|
// value ModelHasNothingServing exists to catch, and which, emitted
|
||||||
|
// only for replicas that were Ready, it never saw.
|
||||||
|
if h.Desired > 0 || ready[h.Model] > 0 {
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.replicaReady, prometheus.GaugeValue,
|
||||||
|
float64(ready[h.Model]), h.Model)
|
||||||
|
}
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.modelDesired, prometheus.GaugeValue, float64(h.Desired), h.Model)
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.modelFailures, prometheus.GaugeValue, float64(h.Failures), h.Model)
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.modelParked, prometheus.GaugeValue, b2f(h.Parked), h.Model)
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.modelUnplaceable, prometheus.GaugeValue, b2f(h.Unplaceable), h.Model)
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.modelPreempted, prometheus.CounterValue, float64(h.Preempted), h.Model, h.Class)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func b2f(b bool) float64 {
|
||||||
|
if b {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// QuotaSource adds the tenancy tree's capacity quotas. Asked for by type
|
||||||
|
// assertion, like HealthSource.
|
||||||
|
type QuotaSource interface {
|
||||||
|
QuotaUse() []reconciler.QuotaUse
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Collector) collectQuotas(ch chan<- prometheus.Metric) {
|
||||||
|
qs, ok := c.src.(QuotaSource)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, q := range qs.QuotaUse() {
|
||||||
|
for _, r := range []struct {
|
||||||
|
name string
|
||||||
|
holds int
|
||||||
|
gua, lim *int
|
||||||
|
}{
|
||||||
|
{"gpus", q.Holds.GPUs, q.Guarantee.GPUs, q.Limit.GPUs},
|
||||||
|
{"cpu_cores", q.Holds.CPUCores, q.Guarantee.CPUCores, q.Limit.CPUCores},
|
||||||
|
{"memory_gb", q.Holds.MemoryGB, q.Guarantee.MemoryGB, q.Limit.MemoryGB},
|
||||||
|
} {
|
||||||
|
if r.gua == nil && r.lim == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.capacityHolds, prometheus.GaugeValue, float64(r.holds), q.Scope, r.name)
|
||||||
|
if r.gua != nil {
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.capacityGuarantee, prometheus.GaugeValue, float64(*r.gua), q.Scope, r.name)
|
||||||
|
}
|
||||||
|
if r.lim != nil {
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.capacityLimit, prometheus.GaugeValue, float64(*r.lim), q.Scope, r.name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sensorFor is a GPU's fresh sensor reading, if the node sent one.
|
||||||
|
func sensorFor(l *inventory.Load, index int, now time.Time) (inventory.GPUSensor, bool) {
|
||||||
|
if l == nil || now.Sub(l.At) >= loadFresh {
|
||||||
|
return inventory.GPUSensor{}, false
|
||||||
|
}
|
||||||
|
for _, g := range l.GPUs {
|
||||||
|
if g.Index == index {
|
||||||
|
return g, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return inventory.GPUSensor{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor, labels []string) {
|
||||||
|
for _, v := range []struct {
|
||||||
|
d *prometheus.Desc
|
||||||
|
val float64
|
||||||
|
}{{c.gpuTemp, s.TempC}, {c.gpuMemTemp, s.MemoryTempC}, {c.gpuPower, s.PowerW}, {c.gpuPowerLimit, s.PowerLimitW}} {
|
||||||
|
if v.val > 0 {
|
||||||
|
ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, labels...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, reason := range s.Throttle {
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.gpuThrottled, prometheus.GaugeValue, 1, append(labels, reason)...)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -196,3 +196,35 @@ func TestEveryIBPortIsReportedWithItsCounters(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type healthSource struct {
|
||||||
|
fakeSource
|
||||||
|
health []reconciler.ModelHealth
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h healthSource) ModelHealth() []reconciler.ModelHealth { return h.health }
|
||||||
|
func (h healthSource) IsLeader() bool { return true }
|
||||||
|
|
||||||
|
func TestAModelThatShouldRunPublishesZeroReplicasReady(t *testing.T) {
|
||||||
|
// ModelHasNothingServing alerts on llmgw_model_replicas_ready == 0, and
|
||||||
|
// the series was only ever published while a replica was ready — so the
|
||||||
|
// alert could not fire for the case it was written for.
|
||||||
|
src := healthSource{health: []reconciler.ModelHealth{
|
||||||
|
{Model: "wanted", Desired: 1, Failures: 5, Parked: true, Reason: "boom"},
|
||||||
|
{Model: "stopped", Desired: 0},
|
||||||
|
}}
|
||||||
|
out := gather(t, src)
|
||||||
|
for _, want := range []string{
|
||||||
|
"llmgw_model_replicas_ready model=wanted -> 0",
|
||||||
|
"llmgw_model_parked model=wanted -> 1",
|
||||||
|
"llmgw_model_start_failures model=wanted -> 5",
|
||||||
|
"llmgw_control_plane_leader -> 1",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(out, want) {
|
||||||
|
t.Errorf("missing %s", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(out, "llmgw_model_replicas_ready model=stopped") {
|
||||||
|
t.Error("a stopped model publishes a ready count, so the alert would fire for it")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
package fleetmetrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) {
|
||||||
|
gpuNode := inventory.Node{ID: "g1", State: inventory.Ready,
|
||||||
|
GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, Health: discovery.GPUHealth{TemperatureC: 40}}},
|
||||||
|
Load: &inventory.Load{At: time.Now(), CPUTempC: 58, CPUPowerW: 65,
|
||||||
|
GPUs: []inventory.GPUSensor{{Index: 0, TempC: 88, PowerW: 698, PowerLimitW: 700,
|
||||||
|
Throttle: []string{"sw_power_cap", "sw_thermal"}}}}}
|
||||||
|
vm := inventory.Node{ID: "vm1", State: inventory.Ready, Load: &inventory.Load{At: time.Now(), CPUPercent: 3}}
|
||||||
|
out := gather(t, fakeSource{nodes: []inventory.Node{gpuNode, vm}})
|
||||||
|
|
||||||
|
for _, want := range []string{
|
||||||
|
"llmgw_node_cpu_temperature_celsius node=g1 -> 58",
|
||||||
|
"llmgw_node_cpu_power_watts node=g1 -> 65",
|
||||||
|
"llmgw_gpu_power_watts class=h100-80g gpu=0 node=g1 -> 698",
|
||||||
|
"llmgw_gpu_power_limit_watts class=h100-80g gpu=0 node=g1 -> 700",
|
||||||
|
"llmgw_gpu_throttled class=h100-80g gpu=0 node=g1 reason=sw_thermal -> 1",
|
||||||
|
// The fresh reading, not the 40 °C from registration.
|
||||||
|
"llmgw_gpu_temperature_celsius class=h100-80g gpu=0 node=g1 -> 88",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(out, want) {
|
||||||
|
t.Errorf("missing %q", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, absent := range []string{"node=vm1 -> 0", "llmgw_node_inlet_temperature_celsius", "llmgw_gpu_memory_temperature_celsius"} {
|
||||||
|
for _, line := range strings.Split(out, "\n") {
|
||||||
|
if strings.Contains(line, absent) && (strings.Contains(line, "temperature") || strings.Contains(line, "power")) {
|
||||||
|
t.Errorf("published what was not measured: %s", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -79,6 +79,9 @@ type Node struct {
|
|||||||
// it, which is treated as no constraint — the same as every node before
|
// it, which is treated as no constraint — the same as every node before
|
||||||
// this field existed.
|
// this field existed.
|
||||||
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
|
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
|
||||||
|
// CPUCount is the CPUs the agent may run on. Zero is an agent too old to
|
||||||
|
// say, which, like memory, constrains nothing.
|
||||||
|
CPUCount int `json:"cpu_count,omitempty"`
|
||||||
|
|
||||||
// Weights are the model files this node can see, however they got there.
|
// Weights are the model files this node can see, however they got there.
|
||||||
// Empty means the node was not asked to look, which is not the same as
|
// Empty means the node was not asked to look, which is not the same as
|
||||||
@@ -301,6 +304,21 @@ type Load struct {
|
|||||||
MemAvailBytes int64
|
MemAvailBytes int64
|
||||||
// Shards is each engine's use, by shard ID.
|
// Shards is each engine's use, by shard ID.
|
||||||
Shards map[string]ShardLoad
|
Shards map[string]ShardLoad
|
||||||
|
// What the node's sensors say; zero is not measured.
|
||||||
|
CPUTempC float64
|
||||||
|
CPUPowerW float64
|
||||||
|
InletTempC float64
|
||||||
|
GPUs []GPUSensor
|
||||||
|
}
|
||||||
|
|
||||||
|
// GPUSensor is one card's heat and power, and what is throttling it.
|
||||||
|
type GPUSensor struct {
|
||||||
|
Index int `json:"index"`
|
||||||
|
TempC float64 `json:"temp_c,omitempty"`
|
||||||
|
MemoryTempC float64 `json:"memory_temp_c,omitempty"`
|
||||||
|
PowerW float64 `json:"power_w,omitempty"`
|
||||||
|
PowerLimitW float64 `json:"power_limit_w,omitempty"`
|
||||||
|
Throttle []string `json:"throttle,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ShardLoad is one engine's use: cores busy, and resident memory.
|
// ShardLoad is one engine's use: cores busy, and resident memory.
|
||||||
@@ -373,6 +391,9 @@ type Allocation struct {
|
|||||||
// Zero for a profile that declared nothing, which constrains nothing and
|
// Zero for a profile that declared nothing, which constrains nothing and
|
||||||
// is counted as nothing.
|
// is counted as nothing.
|
||||||
MemoryGB int
|
MemoryGB int
|
||||||
|
// CPUCores is the cores this shard was given to compute on, for a shard
|
||||||
|
// with no GPU; zero for one that shares whatever is free.
|
||||||
|
CPUCores int
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store holds the cluster inventory. It is safe for concurrent use.
|
// Store holds the cluster inventory. It is safe for concurrent use.
|
||||||
@@ -392,6 +413,23 @@ func NewStore() *Store {
|
|||||||
return &Store{nodes: map[string]*Node{}, allocs: map[string][]Allocation{}}
|
return &Store{nodes: map[string]*Node{}, allocs: map[string][]Allocation{}}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Clone is a copy to try decisions on without making them: allocations made
|
||||||
|
// on it are its own. Nodes are copied by value; the slices inside them are
|
||||||
|
// shared, and a clone must not write through them.
|
||||||
|
func (s *Store) Clone() *Store {
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
c := NewStore()
|
||||||
|
for id, n := range s.nodes {
|
||||||
|
cp := *n
|
||||||
|
c.nodes[id] = &cp
|
||||||
|
}
|
||||||
|
for id, list := range s.allocs {
|
||||||
|
c.allocs[id] = slices.Clone(list)
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
// Upsert records a node's registration, preserving operator-set state such as
|
// Upsert records a node's registration, preserving operator-set state such as
|
||||||
// a cordon: an agent reconnecting must not silently undo an operator's decision.
|
// a cordon: an agent reconnecting must not silently undo an operator's decision.
|
||||||
// RecordSMChange keeps a port's change of master subnet manager, the last
|
// RecordSMChange keeps a port's change of master subnet manager, the last
|
||||||
|
|||||||
@@ -0,0 +1,562 @@
|
|||||||
|
// Package modelserve takes a model from a registry repository to a replica
|
||||||
|
// answering requests, as one job.
|
||||||
|
//
|
||||||
|
// Adding a model was four commands and a hand-written YAML file: plan it,
|
||||||
|
// fetch it into the mirror, write its catalog entry with a memory figure
|
||||||
|
// somebody guessed, scale it — and then find out whether it loads. This runs
|
||||||
|
// those steps in the control plane, where the mirror, the registry, the
|
||||||
|
// catalog and the fleet already are, and reports each one as it goes, so
|
||||||
|
// whatever asks — ctl, a page — only has to watch.
|
||||||
|
package modelserve
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"regexp"
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Registry reads a repository.
|
||||||
|
type Registry interface {
|
||||||
|
Repo(ctx context.Context, id string) (hf.Repo, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mirror fetches files and reports on them.
|
||||||
|
type Mirror interface {
|
||||||
|
Fetch(ctx context.Context, req mirror.FetchRequest) (mirror.FetchResponse, error)
|
||||||
|
List(ctx context.Context) (mirror.ModelsResponse, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Catalog adds a model.
|
||||||
|
type Catalog interface {
|
||||||
|
Add(ctx context.Context, m catalog.Model, by string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fleet is what the job watches once the model is in the catalog.
|
||||||
|
type Fleet interface {
|
||||||
|
Replicas() []reconciler.Replica
|
||||||
|
ModelHealth() []reconciler.ModelHealth
|
||||||
|
EngineKey(replicaID string) string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deps are what a job needs.
|
||||||
|
type Deps struct {
|
||||||
|
Registry Registry
|
||||||
|
Mirror Mirror
|
||||||
|
Catalog Catalog
|
||||||
|
Fleet Fleet
|
||||||
|
// LlamaVersion is the llama.cpp build the nodes install, which decides
|
||||||
|
// which architectures load.
|
||||||
|
LlamaVersion string
|
||||||
|
HTTP *http.Client
|
||||||
|
Log *slog.Logger
|
||||||
|
// Poll is how often the job looks at the mirror and the fleet.
|
||||||
|
Poll time.Duration
|
||||||
|
// ReadyTimeout bounds the wait for a replica: the node fetching the
|
||||||
|
// weights from the mirror and the engine loading them.
|
||||||
|
ReadyTimeout time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// Request is what to serve.
|
||||||
|
type Request struct {
|
||||||
|
// Repo is the registry repository, owner/name.
|
||||||
|
Repo string `json:"repo"`
|
||||||
|
// Quant picks the file by its quantisation, Q4_K_M unless given.
|
||||||
|
Quant string `json:"quant,omitempty"`
|
||||||
|
// File picks it by name instead.
|
||||||
|
File string `json:"file,omitempty"`
|
||||||
|
// Name is the catalog name; the repository suggests one otherwise.
|
||||||
|
Name string `json:"name,omitempty"`
|
||||||
|
Aliases []string `json:"aliases,omitempty"`
|
||||||
|
Description string `json:"description,omitempty"`
|
||||||
|
// Replicas to run; 0 makes it on demand, loaded when first asked for.
|
||||||
|
Replicas int `json:"replicas"`
|
||||||
|
// CtxPool is llama.cpp's --ctx-size; ContextLength what one request may
|
||||||
|
// use. They default to 16384 and 4096, what four parallel slots divide.
|
||||||
|
CtxPool int `json:"ctx_pool,omitempty"`
|
||||||
|
ContextLength int `json:"context_length,omitempty"`
|
||||||
|
// CPUCores is what a replica computes on; zero takes half the smallest
|
||||||
|
// node it fits on.
|
||||||
|
CPUCores int `json:"cpu_cores,omitempty"`
|
||||||
|
// By is who asked, for the catalog's record.
|
||||||
|
By string `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// The states a job and its steps can be in.
|
||||||
|
const (
|
||||||
|
Running = "running"
|
||||||
|
Done = "done"
|
||||||
|
Failed = "failed"
|
||||||
|
Skipped = "skipped"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Step is one stage of a job.
|
||||||
|
type Step struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Detail string `json:"detail,omitempty"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Job is one model being served, and where it stands.
|
||||||
|
type Job struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Request Request `json:"request"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
Model string `json:"model,omitempty"`
|
||||||
|
Steps []Step `json:"steps"`
|
||||||
|
Started time.Time `json:"started"`
|
||||||
|
// Reply is what the model said to the smoke test, and TokensPerSecond
|
||||||
|
// how fast it said it.
|
||||||
|
Reply string `json:"reply,omitempty"`
|
||||||
|
TokensPerSecond float64 `json:"tokens_per_second,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Server runs jobs and remembers them.
|
||||||
|
type Server struct {
|
||||||
|
deps Deps
|
||||||
|
mu sync.Mutex
|
||||||
|
jobs map[string]*Job
|
||||||
|
seq int
|
||||||
|
}
|
||||||
|
|
||||||
|
// New builds a server.
|
||||||
|
func New(d Deps) *Server {
|
||||||
|
if d.Poll <= 0 {
|
||||||
|
d.Poll = 2 * time.Second
|
||||||
|
}
|
||||||
|
if d.ReadyTimeout <= 0 {
|
||||||
|
d.ReadyTimeout = 30 * time.Minute
|
||||||
|
}
|
||||||
|
if d.HTTP == nil {
|
||||||
|
d.HTTP = &http.Client{Timeout: 5 * time.Minute}
|
||||||
|
}
|
||||||
|
if d.Log == nil {
|
||||||
|
d.Log = slog.Default()
|
||||||
|
}
|
||||||
|
return &Server{deps: d, jobs: map[string]*Job{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
var repoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||||
|
|
||||||
|
// Start begins a job and returns at once; the job runs on its own.
|
||||||
|
func (s *Server) Start(req Request) (Job, error) {
|
||||||
|
req.Repo = strings.Trim(strings.TrimSpace(req.Repo), "/")
|
||||||
|
if !repoPattern.MatchString(req.Repo) {
|
||||||
|
return Job{}, fmt.Errorf("%q is not a repository; it looks like owner/name", req.Repo)
|
||||||
|
}
|
||||||
|
if req.Replicas < 0 {
|
||||||
|
return Job{}, errors.New("replicas cannot be negative")
|
||||||
|
}
|
||||||
|
if req.Quant == "" && req.File == "" {
|
||||||
|
req.Quant = "Q4_K_M"
|
||||||
|
}
|
||||||
|
if req.CtxPool <= 0 {
|
||||||
|
req.CtxPool = 16384
|
||||||
|
}
|
||||||
|
if req.ContextLength <= 0 {
|
||||||
|
req.ContextLength = 4096
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
s.seq++
|
||||||
|
j := &Job{ID: fmt.Sprintf("serve-%d-%d", time.Now().Unix(), s.seq), Request: req,
|
||||||
|
State: Running, Started: time.Now()}
|
||||||
|
s.jobs[j.ID] = j
|
||||||
|
s.mu.Unlock()
|
||||||
|
go s.run(j)
|
||||||
|
return s.snapshot(j), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get returns a job as it stands.
|
||||||
|
func (s *Server) Get(id string) (Job, bool) {
|
||||||
|
s.mu.Lock()
|
||||||
|
j, ok := s.jobs[id]
|
||||||
|
s.mu.Unlock()
|
||||||
|
if !ok {
|
||||||
|
return Job{}, false
|
||||||
|
}
|
||||||
|
return s.snapshot(j), true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) snapshot(j *Job) Job {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
out := *j
|
||||||
|
out.Steps = slices.Clone(j.Steps)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// step starts a new step, or updates the detail of the current one.
|
||||||
|
func (s *Server) step(j *Job, name, state, detail string) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if n := len(j.Steps); n > 0 && j.Steps[n-1].Name == name {
|
||||||
|
j.Steps[n-1].State, j.Steps[n-1].Detail, j.Steps[n-1].At = state, detail, time.Now()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
j.Steps = append(j.Steps, Step{Name: name, State: state, Detail: detail, At: time.Now()})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) fail(j *Job, name string, err error) {
|
||||||
|
s.step(j, name, Failed, err.Error())
|
||||||
|
s.mu.Lock()
|
||||||
|
j.State, j.Error = Failed, err.Error()
|
||||||
|
s.mu.Unlock()
|
||||||
|
s.deps.Log.Warn("serving a model failed", "job", j.ID, "repo", j.Request.Repo, "step", name, "err", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) run(j *Job) {
|
||||||
|
ctx := context.Background()
|
||||||
|
req := j.Request
|
||||||
|
|
||||||
|
// 1. The registry: which files.
|
||||||
|
s.step(j, "registry", Running, "reading "+req.Repo)
|
||||||
|
repo, err := s.deps.Registry.Repo(ctx, req.Repo)
|
||||||
|
if err != nil {
|
||||||
|
s.fail(j, "registry", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if repo.Gated.Is() {
|
||||||
|
s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", req.Repo))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
set, err := pick(repo, req)
|
||||||
|
if err != nil {
|
||||||
|
s.fail(j, "registry", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
name := req.Name
|
||||||
|
if name == "" {
|
||||||
|
name = hf.ModelName(repo.ID)
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
j.Model = name
|
||||||
|
s.mu.Unlock()
|
||||||
|
s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s",
|
||||||
|
set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision()))
|
||||||
|
|
||||||
|
// 2. The mirror: fetch, and check what arrived against what was published.
|
||||||
|
source, err := s.mirror(ctx, j, repo, set, name)
|
||||||
|
if err != nil {
|
||||||
|
s.fail(j, "mirror", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Whether the nodes' llama.cpp loads it, and how much memory it needs.
|
||||||
|
s.step(j, "check", Running, "reading its header from the mirror")
|
||||||
|
var header gguf.Header
|
||||||
|
if !strings.HasSuffix(source, "/") {
|
||||||
|
v := gguf.Check(ctx, source, s.deps.LlamaVersion)
|
||||||
|
if !v.Servable {
|
||||||
|
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run this: %s", s.deps.LlamaVersion, v.Reason))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
header = v.Header
|
||||||
|
}
|
||||||
|
memGB := int((header.MemoryBytes(set.Bytes, req.CtxPool) + 1<<30 - 1) >> 30)
|
||||||
|
basis := fmt.Sprintf("weights %.1f GB + KV cache %.1f GB for %d tokens + engine",
|
||||||
|
gb(set.Bytes), gb(int64(req.CtxPool)*header.KVBytesPerToken()), req.CtxPool)
|
||||||
|
if memGB == 0 {
|
||||||
|
// A set of shards, or a header that did not say: weights plus half
|
||||||
|
// again plus two, the rough figure from before shapes were read.
|
||||||
|
memGB = int(gb(set.Bytes)*1.5) + 2
|
||||||
|
basis = "weights plus half again plus two; the header gave no shape"
|
||||||
|
}
|
||||||
|
s.step(j, "check", Done, fmt.Sprintf("%s, needs about %d GB (%s)", orUnknown(header.Architecture), memGB, basis))
|
||||||
|
|
||||||
|
cores := req.CPUCores
|
||||||
|
if cores <= 0 {
|
||||||
|
if nc, ok := s.deps.Fleet.(interface{ NodeCPUs(int) []int }); ok {
|
||||||
|
cores = catalog.SuggestCores(nc.NodeCPUs(memGB))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. The catalog.
|
||||||
|
m := catalog.Model{
|
||||||
|
Name: name, Aliases: req.Aliases, Description: req.Description, Source: source,
|
||||||
|
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024,
|
||||||
|
Profiles: []catalog.Profile{{
|
||||||
|
Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100,
|
||||||
|
Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores,
|
||||||
|
}},
|
||||||
|
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
|
||||||
|
}
|
||||||
|
s.step(j, "catalog", Running, "adding "+name)
|
||||||
|
if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil {
|
||||||
|
s.fail(j, "catalog", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if req.Replicas == 0 {
|
||||||
|
s.step(j, "catalog", Done, name+" added on demand: it loads when first asked for")
|
||||||
|
s.finish(j)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.step(j, "catalog", Done, fmt.Sprintf("%s added, %d replica(s)", name, req.Replicas))
|
||||||
|
|
||||||
|
// 5. A replica, ready.
|
||||||
|
rep, err := s.waitReady(ctx, j, name)
|
||||||
|
if err != nil {
|
||||||
|
s.fail(j, "placement", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// 6. Does it answer.
|
||||||
|
s.step(j, "smoke test", Running, "asking "+rep.ID+" to say hello")
|
||||||
|
reply, tps, err := s.smoke(ctx, rep, name)
|
||||||
|
if err != nil {
|
||||||
|
s.fail(j, "smoke test", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
j.Reply, j.TokensPerSecond = reply, tps
|
||||||
|
s.mu.Unlock()
|
||||||
|
s.step(j, "smoke test", Done, fmt.Sprintf("answered at %.1f tokens/s: %q", tps, reply))
|
||||||
|
s.finish(j)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) finish(j *Job) {
|
||||||
|
s.mu.Lock()
|
||||||
|
j.State = Done
|
||||||
|
s.mu.Unlock()
|
||||||
|
s.deps.Log.Info("model served", "job", j.ID, "repo", j.Request.Repo, "model", j.Model)
|
||||||
|
}
|
||||||
|
|
||||||
|
// pick chooses the file set to serve.
|
||||||
|
func pick(repo hf.Repo, req Request) (hf.FileSet, error) {
|
||||||
|
var weights []hf.File
|
||||||
|
for _, f := range repo.Files {
|
||||||
|
lower := strings.ToLower(f.Path)
|
||||||
|
if !strings.HasSuffix(lower, ".gguf") || strings.Contains(lower, "mmproj") ||
|
||||||
|
strings.Contains(lower, "imatrix") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
weights = append(weights, f)
|
||||||
|
}
|
||||||
|
sets := hf.Group(weights)
|
||||||
|
if len(sets) == 0 {
|
||||||
|
return hf.FileSet{}, fmt.Errorf("%s has no GGUF files; the nodes run llama.cpp, which needs one", repo.ID)
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for _, st := range sets {
|
||||||
|
names = append(names, st.Name)
|
||||||
|
if req.File != "" && (st.Name == req.File || slices.ContainsFunc(st.Files, func(f hf.File) bool { return f.Path == req.File })) {
|
||||||
|
return st, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if req.File == "" {
|
||||||
|
want := strings.ToLower(req.Quant)
|
||||||
|
var match []hf.FileSet
|
||||||
|
for _, st := range sets {
|
||||||
|
if strings.Contains(strings.ToLower(st.Name), want) {
|
||||||
|
match = append(match, st)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(match) == 1 {
|
||||||
|
return match[0], nil
|
||||||
|
}
|
||||||
|
if len(match) > 1 {
|
||||||
|
// Both a split and a joined copy of one quantisation: the joined
|
||||||
|
// one, which is one download to verify rather than several.
|
||||||
|
slices.SortFunc(match, func(a, b hf.FileSet) int { return a.Shards - b.Shards })
|
||||||
|
return match[0], nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return hf.FileSet{}, fmt.Errorf("%s has no file matching %q; it has: %s",
|
||||||
|
repo.ID, or(req.File, req.Quant), strings.Join(names, ", "))
|
||||||
|
}
|
||||||
|
|
||||||
|
// mirror fetches the set into the mirror and returns the source a catalog
|
||||||
|
// entry names: the file, or the directory for a set of shards.
|
||||||
|
func (s *Server) mirror(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name string) (string, error) {
|
||||||
|
version := repo.ShortRevision()
|
||||||
|
s.step(j, "mirror", Running, "asking the mirror for "+set.Name)
|
||||||
|
var base string
|
||||||
|
for _, f := range set.Files {
|
||||||
|
file := mirror.LastPathElement(f.Path)
|
||||||
|
resp, err := s.deps.Mirror.Fetch(ctx, mirror.FetchRequest{
|
||||||
|
Source: repo.DownloadURL("", f.Path), Model: name, Version: version, File: file,
|
||||||
|
})
|
||||||
|
// Already there is not a failure: the same commit fetched before.
|
||||||
|
if err != nil && !strings.Contains(err.Error(), "already") {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
base = resp.URL
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(6 * time.Hour)
|
||||||
|
for {
|
||||||
|
got, err := s.deps.Mirror.List(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var done, total int64
|
||||||
|
present, failed := 0, ""
|
||||||
|
sums := map[string]string{}
|
||||||
|
for _, st := range got.Models {
|
||||||
|
if st.Model != name || st.Version != version {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
done += st.BytesDone
|
||||||
|
total += max(st.BytesTotal, st.BytesDone)
|
||||||
|
switch st.State {
|
||||||
|
case mirror.Present:
|
||||||
|
present++
|
||||||
|
sums[st.File] = st.SHA256
|
||||||
|
case mirror.Failed:
|
||||||
|
failed = st.Err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if failed != "" {
|
||||||
|
return "", fmt.Errorf("the mirror could not fetch it: %s", failed)
|
||||||
|
}
|
||||||
|
if present >= len(set.Files) {
|
||||||
|
// What arrived against what the registry published: a mirror
|
||||||
|
// copies one file to every node, and is the one place a corrupted
|
||||||
|
// download is caught once instead of loaded everywhere.
|
||||||
|
for _, f := range set.Files {
|
||||||
|
file := mirror.LastPathElement(f.Path)
|
||||||
|
if f.SHA256 != "" && sums[file] != "" && !strings.EqualFold(f.SHA256, sums[file]) {
|
||||||
|
return "", fmt.Errorf("%s arrived with sha256 %s, and the registry published %s",
|
||||||
|
file, sums[file][:12], f.SHA256[:12])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
source := base
|
||||||
|
if len(set.Files) > 1 {
|
||||||
|
source = strings.TrimSuffix(source, mirror.LastPathElement(source))
|
||||||
|
}
|
||||||
|
s.step(j, "mirror", Done, fmt.Sprintf("%.1f GB on the mirror, checksums match the registry", gb(total)))
|
||||||
|
return source, nil
|
||||||
|
}
|
||||||
|
if total > 0 {
|
||||||
|
s.step(j, "mirror", Running, fmt.Sprintf("fetching: %.1f of %.1f GB (%d%%)",
|
||||||
|
gb(done), gb(total), done*100/total))
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
return "", errors.New("the mirror had not finished after six hours")
|
||||||
|
}
|
||||||
|
time.Sleep(s.deps.Poll)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitReady waits for a replica of the model to serve, reporting what it is
|
||||||
|
// doing and giving up early when it cannot be placed or keeps failing.
|
||||||
|
func (s *Server) waitReady(ctx context.Context, j *Job, name string) (reconciler.Replica, error) {
|
||||||
|
s.step(j, "placement", Running, "waiting for the control plane to place it")
|
||||||
|
started := time.Now()
|
||||||
|
deadline := started.Add(s.deps.ReadyTimeout)
|
||||||
|
for {
|
||||||
|
for _, h := range s.deps.Fleet.ModelHealth() {
|
||||||
|
if h.Model != name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if h.Parked {
|
||||||
|
return reconciler.Replica{}, fmt.Errorf("it failed to start %d times and is parked: %s", h.Failures, h.Reason)
|
||||||
|
}
|
||||||
|
// A minute's grace from when this step began, not from the job:
|
||||||
|
// the download before it can take hours, and the first pass may
|
||||||
|
// not have run yet.
|
||||||
|
if h.Unplaceable && time.Since(started) > time.Minute {
|
||||||
|
return reconciler.Replica{}, fmt.Errorf("no node can host it:\n%s", h.Placement)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, r := range s.deps.Fleet.Replicas() {
|
||||||
|
if r.Model != name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if r.State == reconciler.Ready {
|
||||||
|
s.step(j, "placement", Done, fmt.Sprintf("%s ready on %s at %s", r.ID,
|
||||||
|
strings.Join(r.NodeIDs(), ","), r.Endpoint))
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
s.step(j, "placement", Running, fmt.Sprintf("%s on %s is %s", r.ID,
|
||||||
|
strings.Join(r.NodeIDs(), ","), r.State))
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
return reconciler.Replica{}, fmt.Errorf("no replica was ready after %s", s.deps.ReadyTimeout)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return reconciler.Replica{}, ctx.Err()
|
||||||
|
case <-time.After(s.deps.Poll):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// smoke sends one short request straight to the replica, with its engine key.
|
||||||
|
func (s *Server) smoke(ctx context.Context, r reconciler.Replica, name string) (string, float64, error) {
|
||||||
|
body, _ := json.Marshal(map[string]any{
|
||||||
|
"model": name,
|
||||||
|
"max_tokens": 32,
|
||||||
|
"messages": []map[string]string{{"role": "user", "content": "Say hello in one short sentence."}},
|
||||||
|
})
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||||
|
strings.TrimRight(r.Endpoint, "/")+"/v1/chat/completions", bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return "", 0, err
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
if k := s.deps.Fleet.EngineKey(r.ID); k != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+k)
|
||||||
|
}
|
||||||
|
start := time.Now()
|
||||||
|
resp, err := s.deps.HTTP.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", 0, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return "", 0, fmt.Errorf("the replica answered %s: %s", resp.Status, strings.TrimSpace(string(b)))
|
||||||
|
}
|
||||||
|
var out struct {
|
||||||
|
Choices []struct {
|
||||||
|
Message struct {
|
||||||
|
Content string `json:"content"`
|
||||||
|
} `json:"message"`
|
||||||
|
} `json:"choices"`
|
||||||
|
Usage struct {
|
||||||
|
CompletionTokens int `json:"completion_tokens"`
|
||||||
|
} `json:"usage"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(b, &out); err != nil || len(out.Choices) == 0 {
|
||||||
|
return "", 0, fmt.Errorf("the replica's answer was not a completion: %s", strings.TrimSpace(string(b)))
|
||||||
|
}
|
||||||
|
secs := time.Since(start).Seconds()
|
||||||
|
tps := 0.0
|
||||||
|
if secs > 0 {
|
||||||
|
tps = float64(out.Usage.CompletionTokens) / secs
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(out.Choices[0].Message.Content), tps, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func gb(b int64) float64 { return float64(b) / (1 << 30) }
|
||||||
|
|
||||||
|
func or(a, b string) string {
|
||||||
|
if a != "" {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func orUnknown(arch string) string {
|
||||||
|
if arch == "" {
|
||||||
|
return "architecture not read"
|
||||||
|
}
|
||||||
|
return arch
|
||||||
|
}
|
||||||
@@ -0,0 +1,225 @@
|
|||||||
|
package modelserve
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/binary"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ggufHeader is enough of a GGUF file for its architecture and shape.
|
||||||
|
func ggufHeader() []byte {
|
||||||
|
var b bytes.Buffer
|
||||||
|
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
||||||
|
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
||||||
|
u32 := func(k string, v uint32) { str(k); w(uint32(4)); w(v) }
|
||||||
|
b.WriteString("GGUF")
|
||||||
|
w(uint32(3))
|
||||||
|
w(uint64(0))
|
||||||
|
w(uint64(5))
|
||||||
|
str("general.architecture")
|
||||||
|
w(uint32(8))
|
||||||
|
str("qwen2")
|
||||||
|
u32("qwen2.block_count", 28)
|
||||||
|
u32("qwen2.attention.head_count", 12)
|
||||||
|
u32("qwen2.attention.head_count_kv", 2)
|
||||||
|
u32("qwen2.embedding_length", 1536)
|
||||||
|
return b.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeRegistry struct{ repo hf.Repo }
|
||||||
|
|
||||||
|
func (f fakeRegistry) Repo(context.Context, string) (hf.Repo, error) { return f.repo, nil }
|
||||||
|
|
||||||
|
// fakeMirror serves the header at every file URL and reports each fetched
|
||||||
|
// file present, with the checksum it was told to.
|
||||||
|
type fakeMirror struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
srv *httptest.Server
|
||||||
|
got []mirror.FetchRequest
|
||||||
|
sums map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *fakeMirror) Fetch(_ context.Context, r mirror.FetchRequest) (mirror.FetchResponse, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
m.got = append(m.got, r)
|
||||||
|
return mirror.FetchResponse{URL: m.srv.URL + "/" + r.Model + "/" + r.Version + "/" + r.File}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *fakeMirror) List(context.Context) (mirror.ModelsResponse, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
var out mirror.ModelsResponse
|
||||||
|
for _, r := range m.got {
|
||||||
|
out.Models = append(out.Models, mirror.Status{Model: r.Model, Version: r.Version, File: r.File,
|
||||||
|
State: mirror.Present, BytesDone: 1 << 30, BytesTotal: 1 << 30, SHA256: m.sums[r.File]})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeCatalog struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
added []catalog.Model
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.added = append(c.added, m)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeFleet has a replica ready at engine once the catalog holds the model.
|
||||||
|
type fakeFleet struct {
|
||||||
|
cat *fakeCatalog
|
||||||
|
engine string
|
||||||
|
health []reconciler.ModelHealth
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeFleet) Replicas() []reconciler.Replica {
|
||||||
|
f.cat.mu.Lock()
|
||||||
|
defer f.cat.mu.Unlock()
|
||||||
|
var out []reconciler.Replica
|
||||||
|
for _, m := range f.cat.added {
|
||||||
|
out = append(out, reconciler.Replica{ID: m.Name + "-r1", Model: m.Name,
|
||||||
|
State: reconciler.Ready, Endpoint: f.engine,
|
||||||
|
Shards: []reconciler.Shard{{NodeID: "node01"}}})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
func (f *fakeFleet) ModelHealth() []reconciler.ModelHealth { return f.health }
|
||||||
|
func (f *fakeFleet) EngineKey(string) string { return "k" }
|
||||||
|
|
||||||
|
func setup(t *testing.T, files []hf.File, sums map[string]string) (*Server, *fakeMirror, *fakeCatalog, *fakeFleet) {
|
||||||
|
t.Helper()
|
||||||
|
header := ggufHeader()
|
||||||
|
files0 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
_, _ = w.Write(header)
|
||||||
|
}))
|
||||||
|
t.Cleanup(files0.Close)
|
||||||
|
engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Header.Get("Authorization") != "Bearer k" {
|
||||||
|
w.WriteHeader(http.StatusUnauthorized)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"Hello!"}}],"usage":{"completion_tokens":3}}`))
|
||||||
|
}))
|
||||||
|
t.Cleanup(engine.Close)
|
||||||
|
|
||||||
|
m := &fakeMirror{srv: files0, sums: sums}
|
||||||
|
c := &fakeCatalog{}
|
||||||
|
f := &fakeFleet{cat: c, engine: engine.URL}
|
||||||
|
s := New(Deps{
|
||||||
|
Registry: fakeRegistry{repo: hf.Repo{ID: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Revision: "abcdef123456", Files: files}},
|
||||||
|
Mirror: m, Catalog: c, Fleet: f, Poll: 10 * time.Millisecond, ReadyTimeout: 5 * time.Second,
|
||||||
|
})
|
||||||
|
return s, m, c, f
|
||||||
|
}
|
||||||
|
|
||||||
|
func wait(t *testing.T, s *Server, id string) Job {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(10 * time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if j, _ := s.Get(id); j.State != Running {
|
||||||
|
return j
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatal("the job never finished")
|
||||||
|
return Job{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAModelGoesFromTheRegistryToAnAnsweringReplica(t *testing.T) {
|
||||||
|
files := []hf.File{
|
||||||
|
{Path: "Qwen2.5-1.5B-Instruct-Q8_0.gguf", Bytes: 2 << 30, SHA256: "aa"},
|
||||||
|
{Path: "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "bb"},
|
||||||
|
{Path: "mmproj-Q4_K_M.gguf", Bytes: 1 << 20},
|
||||||
|
}
|
||||||
|
s, m, c, _ := setup(t, files, map[string]string{"Qwen2.5-1.5B-Instruct-Q4_K_M.gguf": "BB"})
|
||||||
|
job, err := s.Start(Request{Repo: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Replicas: 1})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
j := wait(t, s, job.ID)
|
||||||
|
if j.State != Done {
|
||||||
|
t.Fatalf("job %s: %s\n%+v", j.State, j.Error, j.Steps)
|
||||||
|
}
|
||||||
|
if len(m.got) != 1 || m.got[0].File != "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf" || m.got[0].Version != "abcdef12" {
|
||||||
|
t.Errorf("fetched %+v; expected the Q4_K_M file at its revision", m.got)
|
||||||
|
}
|
||||||
|
mod := c.added[0]
|
||||||
|
if mod.Name != "qwen2.5-1.5b-instruct" || mod.Profiles[0].MinMemoryGB == 0 || mod.Scaling.Min != 1 {
|
||||||
|
t.Errorf("catalog entry %+v", mod)
|
||||||
|
}
|
||||||
|
if j.Reply != "Hello!" {
|
||||||
|
t.Errorf("the smoke test's answer was %q", j.Reply)
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for _, st := range j.Steps {
|
||||||
|
names = append(names, st.Name)
|
||||||
|
}
|
||||||
|
if strings.Join(names, ",") != "registry,mirror,check,catalog,placement,smoke test" {
|
||||||
|
t.Errorf("steps %v", names)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAQuantTheRepositoryDoesNotHaveIsNamedWithWhatItHas(t *testing.T) {
|
||||||
|
s, _, c, _ := setup(t, []hf.File{{Path: "m-Q8_0.gguf", Bytes: 1}}, nil)
|
||||||
|
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
|
||||||
|
j := wait(t, s, job.ID)
|
||||||
|
if j.State != Failed || !strings.Contains(j.Error, "m-Q8_0.gguf") {
|
||||||
|
t.Errorf("%s: %s", j.State, j.Error)
|
||||||
|
}
|
||||||
|
if len(c.added) != 0 {
|
||||||
|
t.Error("something was added to the catalog")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestADownloadThatDoesNotMatchTheRegistryIsNotServed(t *testing.T) {
|
||||||
|
files := []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "aaaaaaaaaaaaaaaa"}}
|
||||||
|
s, _, c, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "bbbbbbbbbbbbbbbb"})
|
||||||
|
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
|
||||||
|
j := wait(t, s, job.ID)
|
||||||
|
if j.State != Failed || !strings.Contains(j.Error, "registry published") {
|
||||||
|
t.Errorf("%s: %s", j.State, j.Error)
|
||||||
|
}
|
||||||
|
if len(c.added) != 0 {
|
||||||
|
t.Error("a corrupted download reached the catalog")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAParkedModelStopsTheWait(t *testing.T) {
|
||||||
|
s, _, _, f := setup(t, []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30}}, nil)
|
||||||
|
f.health = []reconciler.ModelHealth{{Model: "qwen2.5-1.5b-instruct", Parked: true, Failures: 5, Reason: "unknown model architecture"}}
|
||||||
|
// A fleet where it never becomes ready, only parked.
|
||||||
|
s.deps.Fleet = parkedFleet{f}
|
||||||
|
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
|
||||||
|
j := wait(t, s, job.ID)
|
||||||
|
if j.State != Failed || !strings.Contains(j.Error, "parked") {
|
||||||
|
t.Errorf("%s: %s", j.State, j.Error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type parkedFleet struct{ *fakeFleet }
|
||||||
|
|
||||||
|
func (parkedFleet) Replicas() []reconciler.Replica { return nil }
|
||||||
|
|
||||||
|
func TestARepositoryThatIsNotOneIsRefused(t *testing.T) {
|
||||||
|
s := New(Deps{})
|
||||||
|
for _, r := range []string{"", "noslash", "../etc/passwd", "a/b/c"} {
|
||||||
|
if _, err := s.Start(Request{Repo: r}); err == nil {
|
||||||
|
t.Errorf("%q accepted", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -18,6 +18,7 @@ package modelsvc
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -86,6 +87,9 @@ func (s *Service) Sources(ctx context.Context) (map[string][]catalog.Model, erro
|
|||||||
if m.Source != "" {
|
if m.Source != "" {
|
||||||
out[m.Source] = append(out[m.Source], m)
|
out[m.Source] = append(out[m.Source], m)
|
||||||
}
|
}
|
||||||
|
for _, ad := range m.Adapters {
|
||||||
|
out[ad.Source] = append(out[ad.Source], m)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
@@ -104,6 +108,16 @@ func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error {
|
|||||||
if s.Declared(m.Name) {
|
if s.Declared(m.Name) {
|
||||||
return fmt.Errorf("model %q is declared in configuration; edit that rather than adding a second definition", m.Name)
|
return fmt.Errorf("model %q is declared in configuration; edit that rather than adding a second definition", m.Name)
|
||||||
}
|
}
|
||||||
|
// A model with no GPU is all host memory, and one that declares none
|
||||||
|
// fits everywhere as far as placement can tell — however many of it
|
||||||
|
// there already are. Required of what is added now; models stored before
|
||||||
|
// this still load, and say what they are missing.
|
||||||
|
for _, p := range m.Profiles {
|
||||||
|
if p.CPUOnly() && p.MinMemoryGB <= 0 {
|
||||||
|
return fmt.Errorf("model %q profile %q runs without a GPU and declares no memory (min_memory_gb): "+
|
||||||
|
"placement could not tell when a node is full", m.Name, p.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
if s.check != nil {
|
if s.check != nil {
|
||||||
if err := s.check(ctx, m); err != nil {
|
if err := s.check(ctx, m); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -189,36 +203,115 @@ func (s *Service) Remove(ctx context.Context, name string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Scale changes how many replicas a model runs, without a restart.
|
// Scale changes how many replicas a model added at runtime runs.
|
||||||
//
|
|
||||||
// Only the scaling block moves. Everything else about the model — its source,
|
|
||||||
// its profiles, the engine arguments somebody tuned — is read back from what
|
|
||||||
// was stored and written out again untouched, so changing a number cannot
|
|
||||||
// quietly become a redefinition.
|
|
||||||
//
|
|
||||||
// A model declared in configuration is refused, for the same reason Remove
|
|
||||||
// refuses one: the YAML is read on every start, so the new number would hold
|
|
||||||
// until the next restart and then silently revert. A cluster that disagrees
|
|
||||||
// with its own config file is worse than an error message saying where to edit.
|
|
||||||
func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error) {
|
func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error) {
|
||||||
|
final, err := s.update(ctx, name, by, "change its scaling", func(m *catalog.Model) error {
|
||||||
|
m.Scaling = sc
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
s.log.Info("model scaled", "model", name,
|
||||||
|
"min", final.Scaling.Min, "max", final.Scaling.Max, "by", by)
|
||||||
|
}
|
||||||
|
return final, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resources sets what a model's profile reserves of a node: host memory, and
|
||||||
|
// the CPU cores a replica without a GPU computes on. An empty profile name
|
||||||
|
// sets every profile.
|
||||||
|
func (s *Service) Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error) {
|
||||||
|
if memoryGB < 0 || cpuCores < 0 {
|
||||||
|
return catalog.Model{}, errors.New("memory and cores cannot be negative")
|
||||||
|
}
|
||||||
|
final, err := s.update(ctx, name, by, "change its resources", func(m *catalog.Model) error {
|
||||||
|
found := false
|
||||||
|
for i := range m.Profiles {
|
||||||
|
if profile != "" && m.Profiles[i].Name != profile {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
found = true
|
||||||
|
m.Profiles[i].MinMemoryGB = memoryGB
|
||||||
|
m.Profiles[i].CPUCores = cpuCores
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return fmt.Errorf("model %q has no profile %q", name, profile)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
s.log.Info("model resources set", "model", name, "profile", profile,
|
||||||
|
"memory_gb", memoryGB, "cpu_cores", cpuCores, "by", by)
|
||||||
|
}
|
||||||
|
return final, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetClass sets a model's capacity class: critical, standard or
|
||||||
|
// opportunistic, or empty to go back to what its scaling implies.
|
||||||
|
func (s *Service) SetClass(ctx context.Context, name, class, by string) (catalog.Model, error) {
|
||||||
|
final, err := s.update(ctx, name, by, "change its class", func(m *catalog.Model) error {
|
||||||
|
m.Class = class
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
s.log.Info("model class set", "model", name, "class", final.EffectiveClass(), "by", by)
|
||||||
|
}
|
||||||
|
return final, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetOwner sets the part of the tenancy tree a model's capacity counts
|
||||||
|
// against, org:ID, team:ID or project:ID; empty makes it shared.
|
||||||
|
func (s *Service) SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error) {
|
||||||
|
final, err := s.update(ctx, name, by, "change its owner", func(m *catalog.Model) error {
|
||||||
|
m.Owner = owner
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
s.log.Info("model owner set", "model", name, "owner", owner, "by", by)
|
||||||
|
}
|
||||||
|
return final, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetAdapters replaces the LoRA adapters a model serves. Replicas load the
|
||||||
|
// new set when they next start.
|
||||||
|
func (s *Service) SetAdapters(ctx context.Context, name string, adapters []catalog.Adapter, by string) (catalog.Model, error) {
|
||||||
|
final, err := s.update(ctx, name, by, "change its adapters", func(m *catalog.Model) error {
|
||||||
|
m.Adapters = adapters
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
names := make([]string, 0, len(adapters))
|
||||||
|
for _, a := range adapters {
|
||||||
|
names = append(names, a.Name)
|
||||||
|
}
|
||||||
|
s.log.Info("model adapters set", "model", name, "adapters", names, "by", by)
|
||||||
|
}
|
||||||
|
return final, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// update changes one model added at runtime and keeps the change.
|
||||||
|
//
|
||||||
|
// Built before anything is written, like Add: catalog.New is what applies the
|
||||||
|
// defaults and refuses what does not validate, so the model that comes back
|
||||||
|
// is the one that will run rather than the one that was asked for.
|
||||||
|
func (s *Service) update(ctx context.Context, name, by, what string, change func(*catalog.Model) error) (catalog.Model, error) {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
if s.Declared(name) {
|
if s.Declared(name) {
|
||||||
return catalog.Model{}, fmt.Errorf(
|
return catalog.Model{}, fmt.Errorf(
|
||||||
"model %q is declared in configuration; change its scaling there, or it reverts on the next restart", name)
|
"model %q is declared in configuration; %s there, or it reverts on the next restart", name, what)
|
||||||
}
|
}
|
||||||
|
|
||||||
stored, err := s.storedModels(ctx)
|
stored, err := s.storedModels(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return catalog.Model{}, err
|
return catalog.Model{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
next := append([]catalog.Model{}, s.declared...)
|
next := append([]catalog.Model{}, s.declared...)
|
||||||
found := false
|
found := false
|
||||||
for _, e := range stored {
|
for _, e := range stored {
|
||||||
if e.Name == name {
|
if e.Name == name {
|
||||||
e.Scaling = sc
|
if err := change(&e); err != nil {
|
||||||
|
return catalog.Model{}, err
|
||||||
|
}
|
||||||
found = true
|
found = true
|
||||||
}
|
}
|
||||||
next = append(next, e)
|
next = append(next, e)
|
||||||
@@ -226,20 +319,14 @@ func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by
|
|||||||
if !found {
|
if !found {
|
||||||
return catalog.Model{}, fmt.Errorf("model %q was not added at runtime", name)
|
return catalog.Model{}, fmt.Errorf("model %q was not added at runtime", name)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Built before anything is written, like Add: catalog.New is what applies
|
|
||||||
// the defaults (max follows min when it is left out) and what refuses a
|
|
||||||
// max below a min, so the model that comes back is the one that will run
|
|
||||||
// rather than the one that was asked for.
|
|
||||||
built, err := catalog.New(next)
|
built, err := catalog.New(next)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return catalog.Model{}, fmt.Errorf("scaling %q that way would make the catalog invalid: %w", name, err)
|
return catalog.Model{}, fmt.Errorf("that would make the catalog invalid: %w", err)
|
||||||
}
|
}
|
||||||
final, ok := built.Resolve(name)
|
final, ok := built.Resolve(name)
|
||||||
if !ok {
|
if !ok {
|
||||||
return catalog.Model{}, fmt.Errorf("model %q vanished from the catalog it was just built into", name)
|
return catalog.Model{}, fmt.Errorf("model %q vanished from the catalog it was just built into", name)
|
||||||
}
|
}
|
||||||
|
|
||||||
def, err := json.Marshal(final)
|
def, err := json.Marshal(final)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return catalog.Model{}, err
|
return catalog.Model{}, err
|
||||||
@@ -249,10 +336,7 @@ func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by
|
|||||||
}); err != nil {
|
}); err != nil {
|
||||||
return catalog.Model{}, err
|
return catalog.Model{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
s.apply.ReplaceCatalog(ctx, built)
|
s.apply.ReplaceCatalog(ctx, built)
|
||||||
s.log.Info("model scaled", "model", name,
|
|
||||||
"min", final.Scaling.Min, "max", final.Scaling.Max, "by", by)
|
|
||||||
return final, nil
|
return final, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ func model(name string) catalog.Model {
|
|||||||
return catalog.Model{
|
return catalog.Model{
|
||||||
Name: name, Source: "/models/" + name,
|
Name: name, Source: "/models/" + name,
|
||||||
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||||
Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1}},
|
Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1, MinMemoryGB: 2}},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -337,3 +337,43 @@ func TestMinZeroWithAMaxIsOnDemand(t *testing.T) {
|
|||||||
t.Errorf("the idle timeout was lost; got %v", got.Scaling.IdleTimeout)
|
t.Errorf("the idle timeout was lost; got %v", got.Scaling.IdleTimeout)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestACPUModelAddedNowMustSayHowMuchMemoryItNeeds(t *testing.T) {
|
||||||
|
s, st, _ := newService(t)
|
||||||
|
m := model("bare")
|
||||||
|
m.Profiles[0].MinMemoryGB = 0
|
||||||
|
err := s.Add(t.Context(), m, "tester")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "declares no memory") {
|
||||||
|
t.Fatalf("a CPU model with no memory was added: %v", err)
|
||||||
|
}
|
||||||
|
if len(st.rows) != 0 {
|
||||||
|
t.Error("it was stored anyway")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResourcesAreSetOnAStoredModelAndSurviveARestart(t *testing.T) {
|
||||||
|
s, _, _ := newService(t)
|
||||||
|
if err := s.Add(t.Context(), model("m"), "tester"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := s.Resources(t.Context(), "m", "", 6, 3, "tester")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p := got.Profiles[0]; p.MinMemoryGB != 6 || p.CPUCores != 3 {
|
||||||
|
t.Errorf("profile %+v", p)
|
||||||
|
}
|
||||||
|
if _, err := s.Resources(t.Context(), "m", "nope", 6, 3, "tester"); err == nil {
|
||||||
|
t.Error("a profile the model does not have was accepted")
|
||||||
|
}
|
||||||
|
if _, err := s.Resources(t.Context(), "m", "", -1, 0, "tester"); err == nil {
|
||||||
|
t.Error("negative memory was accepted")
|
||||||
|
}
|
||||||
|
cat, err := s.Load(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m, _ := cat.Resolve("m"); m.Profiles[0].CPUCores != 3 {
|
||||||
|
t.Error("the cores were lost on reload")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
package placement
|
package placement
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||||
@@ -100,3 +101,38 @@ func TestCPUProfileRespectsAvoid(t *testing.T) {
|
|||||||
t.Errorf("placed on %q despite being told to avoid it", got)
|
t.Errorf("placed on %q despite being told to avoid it", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCPUCoresAreReservedAsMemoryIs(t *testing.T) {
|
||||||
|
n := node("n1", "", 0)
|
||||||
|
n.CPUCount = 8
|
||||||
|
st := store(n)
|
||||||
|
s := New(st)
|
||||||
|
p := cpuProfile("cpu")
|
||||||
|
p.CPUCores = 3
|
||||||
|
|
||||||
|
for i := range 2 {
|
||||||
|
res := s.Place([]Profile{p}, nil)
|
||||||
|
if res.Placement == nil {
|
||||||
|
t.Fatalf("replica %d did not fit in 8 cores:\n%s", i+1, res.Explain())
|
||||||
|
}
|
||||||
|
st.Allocate("n1", inventory.Allocation{ShardID: string(rune('a' + i)), CPUCores: 3})
|
||||||
|
}
|
||||||
|
// Six of eight given away: a third would need three.
|
||||||
|
res := s.Place([]Profile{p}, nil)
|
||||||
|
if res.Placement != nil {
|
||||||
|
t.Fatal("a third replica was given cores the node does not have")
|
||||||
|
}
|
||||||
|
if got := res.Explain(); !strings.Contains(got, "has 8 CPUs with 2 free, needs 3") {
|
||||||
|
t.Errorf("the refusal should say how many cores are left:\n%s", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unknown counts constrain nothing: an older agent, or a profile that
|
||||||
|
// reserves no cores.
|
||||||
|
old := node("n2", "", 0)
|
||||||
|
if res := New(store(old)).Place([]Profile{p}, nil); res.Placement == nil {
|
||||||
|
t.Error("a node that does not report its CPUs was excluded")
|
||||||
|
}
|
||||||
|
if res := s.Place([]Profile{cpuProfile("cpu")}, nil); res.Placement == nil {
|
||||||
|
t.Error("a profile reserving no cores was refused")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -39,6 +39,9 @@ type Profile struct {
|
|||||||
// MinDiskGB is the free space needed to hold the weights, checked only on
|
// MinDiskGB is the free space needed to hold the weights, checked only on
|
||||||
// nodes that do not already have them. Zero is unconstrained.
|
// nodes that do not already have them. Zero is unconstrained.
|
||||||
MinDiskGB int
|
MinDiskGB int
|
||||||
|
// CPUCores is the cores a profile with no GPU reserves. Zero is
|
||||||
|
// unconstrained.
|
||||||
|
CPUCores int
|
||||||
// Source is the model's weights, as the engine would be given it. Used to
|
// Source is the model's weights, as the engine would be given it. Used to
|
||||||
// prefer nodes that already hold them; empty asks for no preference.
|
// prefer nodes that already hold them; empty asks for no preference.
|
||||||
Source string
|
Source string
|
||||||
@@ -582,6 +585,18 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Cores, for a replica that computes on them: what is left after the
|
||||||
|
// cores already given to the node's other replicas. Same exemptions
|
||||||
|
// as memory: zero declares nothing, and a node that does not report
|
||||||
|
// its CPUs is an older agent, not one with none.
|
||||||
|
if p.CPUOnly() && p.CPUCores > 0 && n.CPUCount > 0 {
|
||||||
|
free := n.CPUCount - s.claimedCores(n.ID)
|
||||||
|
if free < p.CPUCores {
|
||||||
|
reject(fmt.Sprintf("has %d CPUs with %d free, needs %d", n.CPUCount, free, p.CPUCores))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if p.CPUOnly() {
|
if p.CPUOnly() {
|
||||||
// Every ready node can run it; there is no card to be free.
|
// Every ready node can run it; there is no card to be free.
|
||||||
out = append(out, candidate{node: n})
|
out = append(out, candidate{node: n})
|
||||||
@@ -723,6 +738,15 @@ func (s *Scheduler) claimedMemory(nodeID string) int {
|
|||||||
return total
|
return total
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// claimedCores is the CPU cores already given to replicas on a node.
|
||||||
|
func (s *Scheduler) claimedCores(nodeID string) int {
|
||||||
|
total := 0
|
||||||
|
for _, a := range s.inv.Allocations(nodeID) {
|
||||||
|
total += a.CPUCores
|
||||||
|
}
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
// installing says why an engine is missing, when the node has said.
|
// installing says why an engine is missing, when the node has said.
|
||||||
func installing(n inventory.Node, engine string) string {
|
func installing(n inventory.Node, engine string) string {
|
||||||
for _, p := range n.Provision {
|
for _, p := range n.Provision {
|
||||||
|
|||||||
@@ -0,0 +1,192 @@
|
|||||||
|
// Package quota decides what the tenancy tree is guaranteed of the cluster
|
||||||
|
// and how much of it each part may hold.
|
||||||
|
//
|
||||||
|
// A scope — an organization, team or project — may be guaranteed GPUs, CPU
|
||||||
|
// cores and memory, and limited to at most some amount of each. What a scope
|
||||||
|
// holds is what the replicas of the models it owns, and of the models its
|
||||||
|
// teams and projects own, have reserved. Holding more than its guarantee is
|
||||||
|
// borrowing: allowed while the room is free, and taken back when another
|
||||||
|
// scope within its own guarantee needs it.
|
||||||
|
package quota
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Amount is how much of each resource something holds or needs.
|
||||||
|
type Amount struct {
|
||||||
|
GPUs, CPUCores, MemoryGB int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Plus adds two amounts.
|
||||||
|
func (a Amount) Plus(b Amount) Amount {
|
||||||
|
return Amount{a.GPUs + b.GPUs, a.CPUCores + b.CPUCores, a.MemoryGB + b.MemoryGB}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Over names the first resource in which a is above the governed amounts of
|
||||||
|
// l, with both figures; empty when it is above none.
|
||||||
|
func Over(a Amount, l store.Resources) string {
|
||||||
|
switch {
|
||||||
|
case l.GPUs != nil && a.GPUs > *l.GPUs:
|
||||||
|
return fmt.Sprintf("%d GPUs of %d", a.GPUs, *l.GPUs)
|
||||||
|
case l.CPUCores != nil && a.CPUCores > *l.CPUCores:
|
||||||
|
return fmt.Sprintf("%d cores of %d", a.CPUCores, *l.CPUCores)
|
||||||
|
case l.MemoryGB != nil && a.MemoryGB > *l.MemoryGB:
|
||||||
|
return fmt.Sprintf("%d GB of memory of %d", a.MemoryGB, *l.MemoryGB)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Holding is one replica's hold on the cluster, for working out who holds
|
||||||
|
// what.
|
||||||
|
type Holding struct {
|
||||||
|
Replica string
|
||||||
|
Owner string
|
||||||
|
Amount Amount
|
||||||
|
Since time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// Policy is the quotas and the tree they sit in, read at one moment.
|
||||||
|
type Policy struct {
|
||||||
|
quotas map[string]store.CapacityQuota
|
||||||
|
parents map[string][]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// New builds a policy from stored quotas and each scope's parents.
|
||||||
|
func New(quotas []store.CapacityQuota, parents map[string][]string) *Policy {
|
||||||
|
p := &Policy{quotas: map[string]store.CapacityQuota{}, parents: parents}
|
||||||
|
for _, q := range quotas {
|
||||||
|
p.quotas[Key(q.Scope, q.ScopeID)] = q
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// Key writes a scope the way owners are written: team:ID.
|
||||||
|
func Key(s store.Scope, id string) string { return string(s) + ":" + id }
|
||||||
|
|
||||||
|
// Chain is an owner and every scope above it, nearest first.
|
||||||
|
func (p *Policy) Chain(owner string) []string {
|
||||||
|
if owner == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return append([]string{owner}, p.parents[owner]...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Governing is the quotas along an owner's chain, nearest first.
|
||||||
|
func (p *Policy) Governing(owner string) []store.CapacityQuota {
|
||||||
|
if p == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []store.CapacityQuota
|
||||||
|
for _, s := range p.Chain(owner) {
|
||||||
|
if q, ok := p.quotas[s]; ok {
|
||||||
|
out = append(out, q)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Quotas lists every quota.
|
||||||
|
func (p *Policy) Quotas() []store.CapacityQuota {
|
||||||
|
if p == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]store.CapacityQuota, 0, len(p.quotas))
|
||||||
|
for _, q := range p.quotas {
|
||||||
|
out = append(out, q)
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
return Key(out[i].Scope, out[i].ScopeID) < Key(out[j].Scope, out[j].ScopeID)
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Usage is what every scope holds, its own models' replicas and those of
|
||||||
|
// everything beneath it.
|
||||||
|
func (p *Policy) Usage(hs []Holding) map[string]Amount {
|
||||||
|
out := map[string]Amount{}
|
||||||
|
if p == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
for _, h := range hs {
|
||||||
|
for _, s := range p.Chain(h.Owner) {
|
||||||
|
out[s] = out[s].Plus(h.Amount)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Borrowed is the replicas held beyond a guarantee: of each governed scope's
|
||||||
|
// replicas, oldest first, the ones that take it past what it is guaranteed.
|
||||||
|
// A replica beyond any guarantee along its chain is borrowed.
|
||||||
|
func (p *Policy) Borrowed(hs []Holding) map[string]bool {
|
||||||
|
out := map[string]bool{}
|
||||||
|
if p == nil || len(p.quotas) == 0 {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
sorted := append([]Holding(nil), hs...)
|
||||||
|
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Since.Before(sorted[j].Since) })
|
||||||
|
for key, q := range p.quotas {
|
||||||
|
if !q.Guarantee.Any() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var sum Amount
|
||||||
|
for _, h := range sorted {
|
||||||
|
if !contains(p.Chain(h.Owner), key) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sum = sum.Plus(h.Amount)
|
||||||
|
if Over(sum, q.Guarantee) != "" {
|
||||||
|
out[h.Replica] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Guaranteed reports whether an owner, holding what it holds, is still within
|
||||||
|
// every guarantee along its chain with need added — so it may take back what
|
||||||
|
// others borrowed. An owner with no guarantee anywhere is never guaranteed.
|
||||||
|
func (p *Policy) Guaranteed(owner string, usage map[string]Amount, need Amount) bool {
|
||||||
|
gov := p.Governing(owner)
|
||||||
|
any := false
|
||||||
|
for _, q := range gov {
|
||||||
|
if !q.Guarantee.Any() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
any = true
|
||||||
|
if Over(usage[Key(q.Scope, q.ScopeID)].Plus(need), q.Guarantee) != "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return any
|
||||||
|
}
|
||||||
|
|
||||||
|
// AtLimit says which limit along an owner's chain need would take it past,
|
||||||
|
// empty when none.
|
||||||
|
func (p *Policy) AtLimit(owner string, usage map[string]Amount, need Amount) string {
|
||||||
|
for _, q := range p.Governing(owner) {
|
||||||
|
if !q.Limit.Any() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := Key(q.Scope, q.ScopeID)
|
||||||
|
if over := Over(usage[key].Plus(need), q.Limit); over != "" {
|
||||||
|
return fmt.Sprintf("%s is at its capacity limit (%s)", key, strings.TrimSpace(over))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func contains(list []string, v string) bool {
|
||||||
|
for _, x := range list {
|
||||||
|
if x == v {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package quota
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func n(v int) *int { return &v }
|
||||||
|
|
||||||
|
// acme has two teams; ml is guaranteed 8 GPUs, the org as a whole 12 and
|
||||||
|
// no more than 16.
|
||||||
|
func policy() *Policy {
|
||||||
|
return New([]store.CapacityQuota{
|
||||||
|
{Scope: store.ScopeTeam, ScopeID: "ml", Guarantee: store.Resources{GPUs: n(8)}},
|
||||||
|
{Scope: store.ScopeOrg, ScopeID: "acme", Guarantee: store.Resources{GPUs: n(12)}, Limit: store.Resources{GPUs: n(16)}},
|
||||||
|
}, map[string][]string{
|
||||||
|
"team:ml": {"org:acme"}, "team:web": {"org:acme"},
|
||||||
|
"project:train": {"team:ml", "org:acme"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func at(min int) time.Time { return time.Date(2026, 1, 1, 0, min, 0, 0, time.UTC) }
|
||||||
|
|
||||||
|
func TestAScopeHoldsWhatEverythingBeneathItHolds(t *testing.T) {
|
||||||
|
p := policy()
|
||||||
|
u := p.Usage([]Holding{
|
||||||
|
{Replica: "a", Owner: "project:train", Amount: Amount{GPUs: 4}},
|
||||||
|
{Replica: "b", Owner: "team:web", Amount: Amount{GPUs: 2, CPUCores: 3}},
|
||||||
|
{Replica: "c", Owner: "", Amount: Amount{GPUs: 8}}, // shared: nobody's
|
||||||
|
})
|
||||||
|
if u["team:ml"].GPUs != 4 || u["org:acme"].GPUs != 6 || u["org:acme"].CPUCores != 3 || u["project:train"].GPUs != 4 {
|
||||||
|
t.Errorf("usage %+v", u)
|
||||||
|
}
|
||||||
|
if _, ok := u[""]; ok {
|
||||||
|
t.Error("a shared model was counted against somebody")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWhatGoesPastAGuaranteeIsBorrowedNewestFirst(t *testing.T) {
|
||||||
|
p := policy()
|
||||||
|
b := p.Borrowed([]Holding{
|
||||||
|
{Replica: "old", Owner: "team:ml", Amount: Amount{GPUs: 8}, Since: at(1)},
|
||||||
|
{Replica: "new", Owner: "team:ml", Amount: Amount{GPUs: 4}, Since: at(2)},
|
||||||
|
{Replica: "web", Owner: "team:web", Amount: Amount{GPUs: 4}, Since: at(3)},
|
||||||
|
})
|
||||||
|
// ml: 8 within its 8, then 4 past it. acme: 8 + 4 = 12 within, web's 4
|
||||||
|
// takes the org to 16, past its 12.
|
||||||
|
if b["old"] || !b["new"] || !b["web"] {
|
||||||
|
t.Errorf("borrowed %v", b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAGuaranteeIsOnlyForWhoHasOne(t *testing.T) {
|
||||||
|
p := policy()
|
||||||
|
u := map[string]Amount{"team:ml": {GPUs: 4}, "org:acme": {GPUs: 4}}
|
||||||
|
if !p.Guaranteed("project:train", u, Amount{GPUs: 4}) {
|
||||||
|
t.Error("8 of ml's 8 and 8 of acme's 12 is within both")
|
||||||
|
}
|
||||||
|
if p.Guaranteed("project:train", u, Amount{GPUs: 5}) {
|
||||||
|
t.Error("9 GPUs is past ml's 8")
|
||||||
|
}
|
||||||
|
if p.Guaranteed("", u, Amount{GPUs: 1}) || New(nil, nil).Guaranteed("team:ml", u, Amount{}) {
|
||||||
|
t.Error("an owner with no guarantee is not guaranteed anything")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestALimitStopsGrowthWhereverItIsSet(t *testing.T) {
|
||||||
|
p := policy()
|
||||||
|
u := map[string]Amount{"org:acme": {GPUs: 14}}
|
||||||
|
if why := p.AtLimit("team:web", u, Amount{GPUs: 2}); why != "" {
|
||||||
|
t.Errorf("16 of 16 is at the limit, not past it: %s", why)
|
||||||
|
}
|
||||||
|
why := p.AtLimit("team:web", u, Amount{GPUs: 4})
|
||||||
|
if !strings.Contains(why, "org:acme is at its capacity limit (18 GPUs of 16)") {
|
||||||
|
t.Errorf("the refusal should name the scope and the figures: %q", why)
|
||||||
|
}
|
||||||
|
if p.AtLimit("", u, Amount{GPUs: 99}) != "" {
|
||||||
|
t.Error("a shared model was limited")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
package control
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"log/slog"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// QuotaStore is where capacity quotas and the tenancy tree are kept.
|
||||||
|
type QuotaStore interface {
|
||||||
|
CapacityQuotas(ctx context.Context) ([]store.CapacityQuota, error)
|
||||||
|
ScopeParents(ctx context.Context) (map[string][]string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// quotaRefresh is how stale the reconciler's view of quotas may get. A quota
|
||||||
|
// changed by an operator takes effect within this; the reconciler asks many
|
||||||
|
// times a pass and the store is not asked each time.
|
||||||
|
const quotaRefresh = 30 * time.Second
|
||||||
|
|
||||||
|
// QuotaPolicy returns what the reconciler asks for its quotas: the last
|
||||||
|
// policy read from the store, read again once it is older than quotaRefresh.
|
||||||
|
// A store that fails keeps the last good policy, so a database hiccup does
|
||||||
|
// not drop every guarantee and limit at once.
|
||||||
|
func QuotaPolicy(st QuotaStore, log *slog.Logger) func() *quota.Policy {
|
||||||
|
var mu sync.Mutex
|
||||||
|
var cur *quota.Policy
|
||||||
|
var at time.Time
|
||||||
|
return func() *quota.Policy {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
if cur != nil && time.Since(at) < quotaRefresh {
|
||||||
|
return cur
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
qs, err := st.CapacityQuotas(ctx)
|
||||||
|
if err == nil {
|
||||||
|
var parents map[string][]string
|
||||||
|
if parents, err = st.ScopeParents(ctx); err == nil {
|
||||||
|
cur, at = quota.New(qs, parents), time.Now()
|
||||||
|
return cur
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.Warn("could not read capacity quotas; keeping the last ones", "err", err)
|
||||||
|
at = time.Now() // not again on every call while the store is down
|
||||||
|
return cur
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MeasurementStore is where benchmark results are kept.
|
||||||
|
type MeasurementStore interface {
|
||||||
|
Capacities(ctx context.Context) ([]store.StoredCapacity, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Measurements returns what the reconciler asks for benchmark results, keyed
|
||||||
|
// model/profile, read again once older than quotaRefresh; like QuotaPolicy it
|
||||||
|
// keeps the last good answer when the store fails.
|
||||||
|
func Measurements(st MeasurementStore, log *slog.Logger) func() map[string]reconciler.Measured {
|
||||||
|
var mu sync.Mutex
|
||||||
|
var cur map[string]reconciler.Measured
|
||||||
|
var at time.Time
|
||||||
|
return func() map[string]reconciler.Measured {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
if cur != nil && time.Since(at) < quotaRefresh {
|
||||||
|
return cur
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
at = time.Now()
|
||||||
|
rows, err := st.Capacities(ctx)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("could not read benchmark results; keeping the last ones", "err", err)
|
||||||
|
return cur
|
||||||
|
}
|
||||||
|
next := map[string]reconciler.Measured{}
|
||||||
|
for _, row := range rows {
|
||||||
|
var c bench.Capacity
|
||||||
|
if json.Unmarshal(row.Measurement, &c) != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
next[row.Model+"/"+row.Profile] = reconciler.Measured{
|
||||||
|
MaxConcurrency: c.MaxConcurrencyAtSLO, TokPerSec: c.ThroughputTokPerSec}
|
||||||
|
}
|
||||||
|
cur = next
|
||||||
|
return cur
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,262 @@
|
|||||||
|
package reconciler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Adopt rebuilds replica state from shards an agent reports it is already
|
||||||
|
// running (plan §6.5).
|
||||||
|
//
|
||||||
|
// This is what makes a control plane restart survivable. Without it the
|
||||||
|
// reconciler sees an empty cluster, places duplicates, and the engines already
|
||||||
|
// running keep holding GPUs the inventory believes are free — so the duplicates
|
||||||
|
// fail to allocate, with an error naming the new process rather than the old.
|
||||||
|
//
|
||||||
|
// Shards whose model or profile is no longer in the catalog are returned for
|
||||||
|
// the caller to stop: nothing will ever claim them.
|
||||||
|
func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []AdoptedShard) {
|
||||||
|
defer r.forgetShardsTheAgentDoesNotHave(nodeID, shards)
|
||||||
|
|
||||||
|
byReplica := map[string][]AdoptedShard{}
|
||||||
|
for _, s := range shards {
|
||||||
|
byReplica[s.ReplicaID] = append(byReplica[s.ReplicaID], s)
|
||||||
|
}
|
||||||
|
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
|
||||||
|
for replicaID, group := range byReplica {
|
||||||
|
if existing, known := r.replicas[replicaID]; known {
|
||||||
|
// The other half of a replica we are still assembling. Merge rather
|
||||||
|
// than choose: these shards belong to the same replica as the ones
|
||||||
|
// already adopted, they are simply on a different node whose agent
|
||||||
|
// registered a moment later. Treating them as a competing claim is
|
||||||
|
// what tore multi-node replicas apart on every control plane
|
||||||
|
// restart — one half adopted and failed for being short, the other
|
||||||
|
// stopped as unclaimed when it finally arrived.
|
||||||
|
if !existing.AdoptedAt.IsZero() && len(existing.Shards) < existing.WantShards {
|
||||||
|
r.mergeAdopted(existing, group)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Usually this is an agent reconnecting rather than a control plane
|
||||||
|
// restart, and its status reports keep the state current.
|
||||||
|
//
|
||||||
|
// Not always. Replica ids are derived from the model name and a
|
||||||
|
// counter, so a control plane that restarts and places before the
|
||||||
|
// agents reconnect produces a *different* replica under the same
|
||||||
|
// id, on a freshly allocated port. Skipping the adoption then
|
||||||
|
// leaves the cluster pointing at a port nothing listens on, while
|
||||||
|
// the engine that is actually serving is treated as a stray.
|
||||||
|
//
|
||||||
|
// The fleet is authoritative about what is running (plan §6.5), so
|
||||||
|
// what we merely placed loses to what an agent reports — unless our
|
||||||
|
// copy has a shard that has actually come up, in which case two
|
||||||
|
// things really are running and the one we did not ask for is the
|
||||||
|
// stray.
|
||||||
|
if existing.hasLiveShard() {
|
||||||
|
// Only what we do not recognise. A reconnecting agent reports
|
||||||
|
// the shards we already know, and disowning those would tell it
|
||||||
|
// to stop the very replica that is serving.
|
||||||
|
known := map[string]struct{}{}
|
||||||
|
for _, sh := range existing.Shards {
|
||||||
|
known[sh.ID] = struct{}{}
|
||||||
|
}
|
||||||
|
var strays []AdoptedShard
|
||||||
|
for _, s := range group {
|
||||||
|
if _, ok := known[s.ShardID]; !ok {
|
||||||
|
strays = append(strays, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
unclaimed = append(unclaimed, strays...)
|
||||||
|
r.reservePorts(strays)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
r.log.Warn("a running replica reported by the fleet replaces one we placed",
|
||||||
|
"replica", replicaID, "placed_endpoint", existing.Endpoint)
|
||||||
|
for _, s := range existing.Shards {
|
||||||
|
r.inv.Release(s.ID)
|
||||||
|
}
|
||||||
|
delete(r.replicas, replicaID)
|
||||||
|
}
|
||||||
|
|
||||||
|
first := group[0]
|
||||||
|
m, ok := r.cat.Load().Resolve(first.Model)
|
||||||
|
if !ok {
|
||||||
|
unclaimed = append(unclaimed, group...)
|
||||||
|
r.reservePorts(group)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
prof, ok := m.ProfileByName(first.Profile)
|
||||||
|
if !ok {
|
||||||
|
unclaimed = append(unclaimed, group...)
|
||||||
|
r.reservePorts(group)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
rep := &Replica{
|
||||||
|
ID: replicaID, Model: m.Name, Profile: prof.Name,
|
||||||
|
Fabric: first.Fabric, Pool: first.Pool,
|
||||||
|
// From the agent, which was told it when the shard started. Empty
|
||||||
|
// from an agent too old to report it, which reads as unknown.
|
||||||
|
ProfileFP: first.ProfileFP,
|
||||||
|
Weight: prof.Weight, CreatedAt: r.now(),
|
||||||
|
Endpoint: first.Endpoint,
|
||||||
|
// From the profile, not from what was reported: the point is to
|
||||||
|
// notice when what was reported is short.
|
||||||
|
WantShards: max(prof.Nodes, 1),
|
||||||
|
AdoptedAt: r.now(),
|
||||||
|
}
|
||||||
|
r.mergeAdopted(rep, group)
|
||||||
|
r.replicas[replicaID] = rep
|
||||||
|
|
||||||
|
}
|
||||||
|
return unclaimed
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeAdopted folds shards an agent reports into a replica being reassembled.
|
||||||
|
//
|
||||||
|
// Called both for the first group, which creates the replica, and for every
|
||||||
|
// later group belonging to it, so a replica spanning N nodes is built up over N
|
||||||
|
// registrations. Shard ids already present are skipped: an agent that
|
||||||
|
// reconnects repeats what it is running, and counting those again would carry a
|
||||||
|
// replica past its shard count without a second node ever appearing.
|
||||||
|
func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) {
|
||||||
|
have := make(map[string]struct{}, len(rep.Shards))
|
||||||
|
for _, sh := range rep.Shards {
|
||||||
|
have[sh.ID] = struct{}{}
|
||||||
|
}
|
||||||
|
for _, s := range group {
|
||||||
|
if _, dup := have[s.ShardID]; dup {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
have[s.ShardID] = struct{}{}
|
||||||
|
rep.Shards = append(rep.Shards, Shard{
|
||||||
|
ID: s.ShardID, NodeID: s.NodeID, Rank: s.Rank,
|
||||||
|
GPUIndices: s.GPUIndices, HCAs: s.HCAs,
|
||||||
|
HTTPPort: s.HTTPPort, State: s.State,
|
||||||
|
})
|
||||||
|
// Reclaim the GPUs and the memory, or the next placement hands them
|
||||||
|
// out twice. The memory comes from the catalog rather than the agent:
|
||||||
|
// what a profile asked for is not something a running process reports.
|
||||||
|
r.inv.Allocate(s.NodeID, inventory.Allocation{
|
||||||
|
ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices,
|
||||||
|
MemoryGB: r.profileMemory(rep.Model, rep.Profile),
|
||||||
|
CPUCores: r.adoptedCores(rep.Model, rep.Profile),
|
||||||
|
})
|
||||||
|
// The endpoint comes from the leader, which may be in any of the
|
||||||
|
// groups — the rank 0 node is not necessarily the first to register.
|
||||||
|
if s.Rank == 0 && s.Endpoint != "" {
|
||||||
|
rep.Endpoint = s.Endpoint
|
||||||
|
}
|
||||||
|
}
|
||||||
|
slices.SortFunc(rep.Shards, func(a, b Shard) int { return a.Rank - b.Rank })
|
||||||
|
rep.deriveState(r.now())
|
||||||
|
|
||||||
|
r.log.Info("adopted replica from the fleet",
|
||||||
|
"replica", rep.ID, "model", rep.Model,
|
||||||
|
"shards", len(rep.Shards), "want", rep.WantShards, "state", rep.State)
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgetShardsTheAgentDoesNotHave fails shards we believe are on a node that the
|
||||||
|
// agent, having just told us everything it is running, did not mention.
|
||||||
|
//
|
||||||
|
// The mirror image of adoption, and the half that was missing. Adoption handles
|
||||||
|
// "the agent has something we do not know about"; this handles "we think the
|
||||||
|
// agent has something it does not". Both follow from the same rule: the fleet is
|
||||||
|
// authoritative about what is running (plan §6.5).
|
||||||
|
//
|
||||||
|
// Without it, an engine that dies while its agent survives leaves the control
|
||||||
|
// plane believing a replica is Ready forever. Observed on a real cluster: the
|
||||||
|
// engine was killed, the agent restarted and registered with nothing running,
|
||||||
|
// and the replica stayed Ready pointing at a dead port for as long as anyone
|
||||||
|
// watched. Traffic survived — the breaker took the endpoint out — but the
|
||||||
|
// cluster ran a third short with no sign of it.
|
||||||
|
//
|
||||||
|
// Safe against the obvious race. A registration follows a fresh connection, so
|
||||||
|
// anything we dispatched over a previous one is not running by definition;
|
||||||
|
// there is no window where a shard is legitimately started but unreported.
|
||||||
|
func (r *Reconciler) forgetShardsTheAgentDoesNotHave(nodeID string, running []AdoptedShard) {
|
||||||
|
if nodeID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
reported := make(map[string]struct{}, len(running))
|
||||||
|
for _, s := range running {
|
||||||
|
reported[s.ShardID] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
var lost []string
|
||||||
|
r.mu.Lock()
|
||||||
|
for _, rep := range r.replicas {
|
||||||
|
if rep.State == Stopped {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
changed := false
|
||||||
|
for i := range rep.Shards {
|
||||||
|
sh := &rep.Shards[i]
|
||||||
|
if sh.NodeID != nodeID || sh.State.Terminal() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := reported[sh.ID]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sh.State = supervisor.Failed
|
||||||
|
sh.Err = "the agent reports it is not running"
|
||||||
|
r.inv.Release(sh.ID)
|
||||||
|
changed = true
|
||||||
|
lost = append(lost, sh.ID)
|
||||||
|
}
|
||||||
|
if changed {
|
||||||
|
rep.deriveState(r.now())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
r.mu.Unlock()
|
||||||
|
|
||||||
|
for _, id := range lost {
|
||||||
|
r.log.Warn("a shard we believed was running is not; the agent does not have it",
|
||||||
|
"node", nodeID, "shard", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// reservePorts records ports held by shards nothing owns.
|
||||||
|
//
|
||||||
|
// They are being stopped, but stopping is not instant and the engine keeps its
|
||||||
|
// socket until it exits. Handing the port to a new shard in the meantime makes
|
||||||
|
// that shard fail to bind, which fails its replica, which places another one on
|
||||||
|
// the same port — a loop that does not end on its own.
|
||||||
|
//
|
||||||
|
// Called with the lock held.
|
||||||
|
func (r *Reconciler) reservePorts(shards []AdoptedShard) {
|
||||||
|
for _, s := range shards {
|
||||||
|
if s.HTTPPort > 0 {
|
||||||
|
r.reservedPorts[s.HTTPPort] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// AdoptedShard is one running shard an agent reported at registration.
|
||||||
|
type AdoptedShard struct {
|
||||||
|
NodeID string
|
||||||
|
ReplicaID string
|
||||||
|
ShardID string
|
||||||
|
Rank int
|
||||||
|
GPUIndices []int
|
||||||
|
HCAs []string
|
||||||
|
HTTPPort int
|
||||||
|
State supervisor.State
|
||||||
|
Model string
|
||||||
|
Profile string
|
||||||
|
Pool string
|
||||||
|
Fabric string
|
||||||
|
ProfileFP string
|
||||||
|
// Endpoint is the leader's base URL, set on the rank 0 shard.
|
||||||
|
Endpoint string
|
||||||
|
}
|
||||||
|
|
||||||
|
// adoptedCores is what an adopted shard is taken to hold of its node's CPUs.
|
||||||
|
func (r *Reconciler) adoptedCores(model, profile string) int {
|
||||||
|
p, _ := r.profileOf(model, profile)
|
||||||
|
return cpuCores(p)
|
||||||
|
}
|
||||||
@@ -0,0 +1,251 @@
|
|||||||
|
package reconciler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RecordUsage stamps replicas with when each was last sent a request.
|
||||||
|
//
|
||||||
|
// Replicas absent from the map are left alone rather than zeroed: a router that
|
||||||
|
// has just restarted knows nothing about what happened before it, and treating
|
||||||
|
// that as "never used" would make every replica look abandoned at once —
|
||||||
|
// exactly when a fleet is least able to afford acting on that.
|
||||||
|
func (r *Reconciler) RecordUsage(lastUsed map[string]time.Time) {
|
||||||
|
if len(lastUsed) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
for id, at := range lastUsed {
|
||||||
|
rep, ok := r.replicas[id]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Only forwards. Two routers reporting, or a stale snapshot arriving
|
||||||
|
// late, must not move a replica's last use backwards.
|
||||||
|
if at.After(rep.LastUsedAt) {
|
||||||
|
rep.LastUsedAt = at
|
||||||
|
}
|
||||||
|
}
|
||||||
|
r.noteUsage(lastUsed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IdleFor reports how long a replica has gone without a request, and whether
|
||||||
|
// that is known at all.
|
||||||
|
//
|
||||||
|
// The second return is not a detail. "Idle for an hour" and "nobody has told me
|
||||||
|
// yet" are different states, and anything that acts on idleness — eviction,
|
||||||
|
// most obviously — has to refuse to act on the second.
|
||||||
|
func (r *Reconciler) IdleFor(id string) (time.Duration, bool) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
rep, ok := r.replicas[id]
|
||||||
|
if !ok || rep.LastUsedAt.IsZero() {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
return r.now().Sub(rep.LastUsedAt), true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Want marks an on-demand model as asked for, so the next pass places it.
|
||||||
|
//
|
||||||
|
// Called by whatever fields the request — the router, in practice. It returns
|
||||||
|
// whether the model is one this applies to, so a caller can tell "being loaded"
|
||||||
|
// from "there is genuinely nothing serving this", which are different answers
|
||||||
|
// to give a client.
|
||||||
|
func (r *Reconciler) Want(model string) bool {
|
||||||
|
m, ok := r.cat.Load().Resolve(model)
|
||||||
|
if !ok || !m.Scaling.OnDemand() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
r.mu.Lock()
|
||||||
|
r.wanted[m.Name] = r.now()
|
||||||
|
// A placement that failed before this request says nothing about now:
|
||||||
|
// forgotten, so the pass this triggers decides afresh and Unservable
|
||||||
|
// does not answer from a stale result.
|
||||||
|
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil {
|
||||||
|
delete(r.lastExplain, m.Name)
|
||||||
|
}
|
||||||
|
r.mu.Unlock()
|
||||||
|
// Immediately, not on the next tick. Someone is waiting on this request,
|
||||||
|
// and five seconds of ticker is five seconds they spend staring at nothing.
|
||||||
|
r.Notify()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// desiredFor is how many replicas a model should have right now.
|
||||||
|
//
|
||||||
|
// For an ordinary model that is its configured minimum. For an on-demand one it
|
||||||
|
// is one while somebody is asking and none once they stop, which is the whole
|
||||||
|
// feature: a cluster with more models than GPUs can hold every model in its
|
||||||
|
// catalog and only the wanted ones in its cards.
|
||||||
|
func (r *Reconciler) desiredFor(m catalog.Model) int {
|
||||||
|
base := r.baseFor(m)
|
||||||
|
if base == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
r.mu.Lock()
|
||||||
|
extra := r.extra[m.Name]
|
||||||
|
r.mu.Unlock()
|
||||||
|
return min(base+extra, max(m.Scaling.Max, base))
|
||||||
|
}
|
||||||
|
|
||||||
|
// baseFor is what a model runs without load: its minimum, or for an on-demand
|
||||||
|
// model one while somebody is asking and none once they stop.
|
||||||
|
func (r *Reconciler) baseFor(m catalog.Model) int {
|
||||||
|
if m.IsExternal() {
|
||||||
|
return 0 // served elsewhere: nothing to place, whatever its scaling says
|
||||||
|
}
|
||||||
|
if !m.Scaling.OnDemand() {
|
||||||
|
return m.Scaling.Min
|
||||||
|
}
|
||||||
|
r.mu.Lock()
|
||||||
|
last, asked := r.wanted[m.Name]
|
||||||
|
r.mu.Unlock()
|
||||||
|
if !asked {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if r.now().Sub(last) > r.idleTimeout(m) {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// idleTimeout is how long an on-demand model stays up unasked-for.
|
||||||
|
func (r *Reconciler) idleTimeout(m catalog.Model) time.Duration {
|
||||||
|
if m.Scaling.IdleTimeout > 0 {
|
||||||
|
return m.Scaling.IdleTimeout
|
||||||
|
}
|
||||||
|
return r.cfg.IdleTimeout
|
||||||
|
}
|
||||||
|
|
||||||
|
// noteUsage keeps an on-demand model alive while it is being used.
|
||||||
|
//
|
||||||
|
// Without this a model would be evicted exactly IdleTimeout after the request
|
||||||
|
// that loaded it, however busy it had been since — the timer would measure
|
||||||
|
// time since the first request rather than the last.
|
||||||
|
func (r *Reconciler) noteUsage(lastUsed map[string]time.Time) {
|
||||||
|
byModel := map[string]time.Time{}
|
||||||
|
for id, at := range lastUsed {
|
||||||
|
rep, ok := r.replicas[id]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if at.After(byModel[rep.Model]) {
|
||||||
|
byModel[rep.Model] = at
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for model, at := range byModel {
|
||||||
|
if cur, ok := r.wanted[model]; ok && at.After(cur) {
|
||||||
|
r.wanted[model] = at
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load is what a model's replicas are doing, as the router sees it.
|
||||||
|
type Load struct {
|
||||||
|
Replicas int
|
||||||
|
Running int
|
||||||
|
Waiting int
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordLoad adds or takes away replicas between a model's min and max as its
|
||||||
|
// load says.
|
||||||
|
//
|
||||||
|
// Max was a ceiling nothing reached: every model ran exactly its minimum
|
||||||
|
// however long its queue. Now requests waiting at the engines — every slot of
|
||||||
|
// every replica busy — held for ScaleUpAfter add one replica, and a quiet spell
|
||||||
|
// of ScaleDownAfter takes one of those back. One step at a time, and not while
|
||||||
|
// the last one is still starting: on a small CPU fleet each replica is minutes
|
||||||
|
// of weights and gigabytes of memory, and a spike that is over by the time it
|
||||||
|
// loads should not leave three behind.
|
||||||
|
//
|
||||||
|
// With a benchmark stored for the profile a model runs on, its measured
|
||||||
|
// capacity is used too: more requests per replica than were measured to meet
|
||||||
|
// the SLO is busy, even with the engines' queues empty — an engine will take
|
||||||
|
// requests into its slots long after latency has gone — and a replica is only
|
||||||
|
// taken back when the rest would still be under scaleDownHeadroom of what they
|
||||||
|
// were measured to hold, so scaling down does not land on the edge.
|
||||||
|
func (r *Reconciler) RecordLoad(loads map[string]Load) {
|
||||||
|
now := r.now()
|
||||||
|
// The profile each model's replicas run on, for its measurement; read
|
||||||
|
// before the loop, which holds the lock.
|
||||||
|
running := map[string]string{}
|
||||||
|
r.mu.Lock()
|
||||||
|
for _, rep := range r.replicas {
|
||||||
|
if rep.State.Counts() {
|
||||||
|
running[rep.Model] = rep.Profile
|
||||||
|
}
|
||||||
|
}
|
||||||
|
r.mu.Unlock()
|
||||||
|
measured := map[string]Measured{}
|
||||||
|
for model, profile := range running {
|
||||||
|
if m, ok := r.measured(model, profile); ok {
|
||||||
|
measured[model] = m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, m := range r.cat.Load().Models() {
|
||||||
|
base := r.baseFor(m)
|
||||||
|
r.mu.Lock()
|
||||||
|
if base == 0 || m.Scaling.Max <= base {
|
||||||
|
delete(r.extra, m.Name)
|
||||||
|
delete(r.busySince, m.Name)
|
||||||
|
delete(r.quietSince, m.Name)
|
||||||
|
r.mu.Unlock()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
starting := false
|
||||||
|
for _, rep := range r.replicas {
|
||||||
|
if rep.Model == m.Name && (rep.State == Placing || rep.State == Starting) {
|
||||||
|
starting = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
l := loads[m.Name]
|
||||||
|
perReplica := float64(l.Waiting) / float64(max(l.Replicas, 1))
|
||||||
|
extra := r.extra[m.Name]
|
||||||
|
meas, hasMeas := measured[m.Name]
|
||||||
|
overMeasured := hasMeas &&
|
||||||
|
float64(l.Running+l.Waiting) > float64(meas.MaxConcurrency*max(l.Replicas, 1))
|
||||||
|
fitsInFewer := !hasMeas ||
|
||||||
|
float64(l.Running) <= scaleDownHeadroom*float64(meas.MaxConcurrency*max(l.Replicas-1, 0))
|
||||||
|
switch {
|
||||||
|
case perReplica >= r.cfg.ScaleUpWaiting || overMeasured:
|
||||||
|
delete(r.quietSince, m.Name)
|
||||||
|
since, ok := r.busySince[m.Name]
|
||||||
|
if !ok {
|
||||||
|
r.busySince[m.Name] = now
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if now.Sub(since) >= r.cfg.ScaleUpAfter && base+extra < m.Scaling.Max && !starting {
|
||||||
|
r.extra[m.Name] = extra + 1
|
||||||
|
r.busySince[m.Name] = now
|
||||||
|
r.log.Info("scaling up for load", "model", m.Name, "replicas", base+extra+1,
|
||||||
|
"waiting_per_replica", perReplica, "in_flight", l.Running+l.Waiting,
|
||||||
|
"measured_per_replica", meas.MaxConcurrency, "for", now.Sub(since).Round(time.Second))
|
||||||
|
}
|
||||||
|
case l.Waiting == 0 && extra > 0 && fitsInFewer:
|
||||||
|
delete(r.busySince, m.Name)
|
||||||
|
since, ok := r.quietSince[m.Name]
|
||||||
|
if !ok {
|
||||||
|
r.quietSince[m.Name] = now
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if now.Sub(since) >= r.cfg.ScaleDownAfter {
|
||||||
|
r.extra[m.Name] = extra - 1
|
||||||
|
r.quietSince[m.Name] = now
|
||||||
|
r.log.Info("scaling down after a quiet spell", "model", m.Name,
|
||||||
|
"replicas", base+extra-1, "quiet_for", now.Sub(since).Round(time.Second))
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
delete(r.busySince, m.Name)
|
||||||
|
delete(r.quietSince, m.Name)
|
||||||
|
}
|
||||||
|
r.mu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// scaleDownHeadroom is how full, of their measured capacity, the replicas left
|
||||||
|
// after a scale-down may be. Below one so that the load that was there a
|
||||||
|
// minute ago does not put them straight back over.
|
||||||
|
const scaleDownHeadroom = 0.7
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
package reconciler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Headroom is how many more replicas of a model the cluster has room for now.
|
||||||
|
type Headroom struct {
|
||||||
|
// Current is the replicas the model has that hold capacity: starting or
|
||||||
|
// ready, not those on their way out.
|
||||||
|
Current int
|
||||||
|
// More is how many more could be placed now, at most the limit asked.
|
||||||
|
More int
|
||||||
|
// Nodes is where each of those would go, in order, a replica's nodes
|
||||||
|
// joined when it spans several.
|
||||||
|
Nodes [][]string
|
||||||
|
// Limited is set when More reached the limit asked, so there may be room
|
||||||
|
// for more than that. A CPU profile that declares no memory is always
|
||||||
|
// limited: nothing it asks for runs out.
|
||||||
|
Limited bool
|
||||||
|
// Blocked is why one more than More does not fit, a line per node that
|
||||||
|
// was considered. Empty when Limited.
|
||||||
|
Blocked []string
|
||||||
|
// TokPerSec is what one more replica was measured to serve, on the
|
||||||
|
// profile placement would give it; zero when it was never measured.
|
||||||
|
TokPerSec float64
|
||||||
|
// ByPreempting is how many more, beyond More, would fit by stopping
|
||||||
|
// replicas the model's class may take, and Preempts the models those
|
||||||
|
// replicas belong to: what "room" means for a model that outranks others.
|
||||||
|
ByPreempting int
|
||||||
|
Preempts []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Headroom tries placing up to limit more replicas of a model on a copy of
|
||||||
|
// the inventory, each one's allocation made on the copy before the next is
|
||||||
|
// tried, exactly as the reconciler would place them one pass at a time.
|
||||||
|
//
|
||||||
|
// The scheduler's own decisions, not an estimate beside them: what the page
|
||||||
|
// offers is then what the next passes will do, and a count that the
|
||||||
|
// arithmetic of a second implementation allowed but placement refuses would
|
||||||
|
// leave a model short with nobody told why.
|
||||||
|
func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) {
|
||||||
|
m, ok := r.cat.Load().Resolve(model)
|
||||||
|
if !ok {
|
||||||
|
return Headroom{}, false
|
||||||
|
}
|
||||||
|
var h Headroom
|
||||||
|
var avoid []string
|
||||||
|
r.mu.Lock()
|
||||||
|
for _, rep := range r.replicas {
|
||||||
|
if rep.Model == m.Name && rep.State.Counts() && rep.RetiringAt.IsZero() {
|
||||||
|
h.Current++
|
||||||
|
avoid = append(avoid, rep.NodeIDs()...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
r.mu.Unlock()
|
||||||
|
|
||||||
|
inv := r.inv.Clone()
|
||||||
|
sched := placement.New(inv)
|
||||||
|
pol := r.policy()
|
||||||
|
var usage map[string]quota.Amount
|
||||||
|
if pol != nil {
|
||||||
|
usage = pol.Usage(r.Holdings())
|
||||||
|
}
|
||||||
|
place := func() placement.Result {
|
||||||
|
res := sched.Place(r.orderedProfiles(m), avoid)
|
||||||
|
if res.Placement == nil && len(avoid) > 0 {
|
||||||
|
// As scaleUp does: spreading is a preference, not a rule.
|
||||||
|
res = sched.Place(r.orderedProfiles(m), nil)
|
||||||
|
}
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
// What the model's class may take, in the order preemption would, each
|
||||||
|
// released on the copy only when free room has run out.
|
||||||
|
victims, nextVictim, preempting := r.victims(m), 0, false
|
||||||
|
for h.More+h.ByPreempting < limit {
|
||||||
|
res := place()
|
||||||
|
// Named only once they have made room: candidates tried on the way
|
||||||
|
// to finding there is none would otherwise read as whose replicas a
|
||||||
|
// placement would stop.
|
||||||
|
var tried []string
|
||||||
|
for res.Placement == nil && nextVictim < len(victims) {
|
||||||
|
v := victims[nextVictim]
|
||||||
|
nextVictim++
|
||||||
|
inv.ReleaseReplica(v.ID)
|
||||||
|
tried = append(tried, v.Model)
|
||||||
|
res = place()
|
||||||
|
}
|
||||||
|
if res.Placement != nil && len(tried) > 0 {
|
||||||
|
preempting = true
|
||||||
|
for _, m := range tried {
|
||||||
|
if !slices.Contains(h.Preempts, m) {
|
||||||
|
h.Preempts = append(h.Preempts, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if res.Placement == nil {
|
||||||
|
for _, rej := range res.Rejections {
|
||||||
|
h.Blocked = append(h.Blocked, rej.String())
|
||||||
|
}
|
||||||
|
slices.Sort(h.Blocked)
|
||||||
|
h.Blocked = slices.Compact(h.Blocked)
|
||||||
|
return h, true
|
||||||
|
}
|
||||||
|
// A quota limit is as much an edge as a full node.
|
||||||
|
if pol != nil && m.Owner != "" {
|
||||||
|
need := placementNeed(m, res.Placement)
|
||||||
|
if why := pol.AtLimit(m.Owner, usage, need); why != "" {
|
||||||
|
h.Blocked = []string{why}
|
||||||
|
return h, true
|
||||||
|
}
|
||||||
|
for _, s := range pol.Chain(m.Owner) {
|
||||||
|
usage[s] = usage[s].Plus(need)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mem, cores := 0, 0
|
||||||
|
if prof, ok := m.ProfileByName(res.Placement.Profile.Name); ok {
|
||||||
|
mem, cores = prof.MinMemoryGB, cpuCores(prof)
|
||||||
|
}
|
||||||
|
if h.More == 0 {
|
||||||
|
if meas, ok := r.measured(m.Name, res.Placement.Profile.Name); ok {
|
||||||
|
h.TokPerSec = meas.TokPerSec
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var nodes []string
|
||||||
|
for _, a := range res.Placement.Nodes {
|
||||||
|
inv.Allocate(a.NodeID, inventory.Allocation{
|
||||||
|
ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, CPUCores: cores,
|
||||||
|
})
|
||||||
|
nodes = append(nodes, a.NodeID)
|
||||||
|
}
|
||||||
|
avoid = append(avoid, nodes...)
|
||||||
|
// Once anything was stopped, what fits after is counted as fitting
|
||||||
|
// by preempting: it may be sitting in the room that was freed.
|
||||||
|
if preempting {
|
||||||
|
h.ByPreempting++
|
||||||
|
} else {
|
||||||
|
h.Nodes = append(h.Nodes, nodes)
|
||||||
|
h.More++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
h.Limited = true
|
||||||
|
return h, true
|
||||||
|
}
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
package reconciler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHeadroomIsWhatPlacementWouldDoNext(t *testing.T) {
|
||||||
|
// Two 8-GPU nodes and a 4-GPU profile: four replicas in all, one placed.
|
||||||
|
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
|
||||||
|
h.r.Reconcile(t.Context())
|
||||||
|
h.markReady()
|
||||||
|
|
||||||
|
got, ok := h.r.Headroom("llama", 10)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("the model was not found")
|
||||||
|
}
|
||||||
|
if got.Current != 1 || got.More != 3 || got.Limited {
|
||||||
|
t.Fatalf("headroom %+v, expected 1 placed and room for 3 more", got)
|
||||||
|
}
|
||||||
|
// Spread first: the next one goes to the node without one.
|
||||||
|
if first := got.Nodes[0][0]; first == h.r.Replicas()[0].NodeIDs()[0] {
|
||||||
|
t.Errorf("the next replica would stack on %s beside the first", first)
|
||||||
|
}
|
||||||
|
if len(got.Blocked) == 0 || !strings.Contains(strings.Join(got.Blocked, "\n"), "GPU") {
|
||||||
|
t.Errorf("no reason given for the fifth: %v", got.Blocked)
|
||||||
|
}
|
||||||
|
// Tried on a copy: nothing was taken from the real inventory.
|
||||||
|
if free := len(h.inv.FreeGPUs("1")) + len(h.inv.FreeGPUs("2")); free != 12 {
|
||||||
|
t.Errorf("%d GPUs free after asking, expected the 12 the one replica leaves", free)
|
||||||
|
}
|
||||||
|
|
||||||
|
capped, _ := h.r.Headroom("llama", 2)
|
||||||
|
if capped.More != 2 || !capped.Limited || capped.Blocked != nil {
|
||||||
|
t.Errorf("asked for at most 2: %+v", capped)
|
||||||
|
}
|
||||||
|
if _, ok := h.r.Headroom("nope", 5); ok {
|
||||||
|
t.Error("an unknown model has headroom")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHeadroomCountsMemoryAlreadyClaimed(t *testing.T) {
|
||||||
|
cpu := func(id string) inventory.Node {
|
||||||
|
return inventory.Node{ID: id, MgmtAddr: "10.0.0." + id, State: inventory.Ready, MemoryTotalMB: 16 * 1024}
|
||||||
|
}
|
||||||
|
m := catalog.Model{Name: "small", Source: "/m.gguf", Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||||
|
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 6}}}
|
||||||
|
h := newHarness(t, m, cpu("a"), cpu("b"))
|
||||||
|
h.r.Reconcile(t.Context())
|
||||||
|
|
||||||
|
// 16 GB each at 6 GB a replica: two per node, one of them taken.
|
||||||
|
got, _ := h.r.Headroom("small", 10)
|
||||||
|
if got.Current != 1 || got.More != 3 {
|
||||||
|
t.Errorf("headroom %+v, expected room for 3 more", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReservedCoresAreTheEnginesThreadsAndBoundTheRoom(t *testing.T) {
|
||||||
|
cpu := inventory.Node{ID: "a", MgmtAddr: "10.0.0.1", State: inventory.Ready, MemoryTotalMB: 64 * 1024, CPUCount: 8}
|
||||||
|
m := catalog.Model{Name: "coder", Source: "/m.gguf", Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||||
|
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 6, CPUCores: 3}}}
|
||||||
|
h := newHarness(t, m, cpu)
|
||||||
|
h.r.Reconcile(t.Context())
|
||||||
|
|
||||||
|
started := h.disp.startedShards()
|
||||||
|
if len(started) != 1 || started[0].Spec.Threads != 3 {
|
||||||
|
t.Fatalf("the engine was not told its 3 cores: %+v", started)
|
||||||
|
}
|
||||||
|
if a := h.inv.Allocations("a"); len(a) != 1 || a[0].CPUCores != 3 {
|
||||||
|
t.Errorf("the cores were not reserved: %+v", a)
|
||||||
|
}
|
||||||
|
// Memory would allow nine more; cores allow one: 8 - 3 = 5, then 2.
|
||||||
|
got, _ := h.r.Headroom("coder", 10)
|
||||||
|
if got.More != 1 {
|
||||||
|
t.Errorf("room for %d more, expected 1 by cores: %+v", got.More, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
Loaded 100 of 203 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user