Compare commits
No files matched your search
@@ -39,6 +39,14 @@ jobs:
|
||||
- name: Build
|
||||
run: go build ./...
|
||||
|
||||
# llmgw-kb runs on people's own machines, Windows included, which no
|
||||
# test here runs on: building it for each catches a Unix-only call.
|
||||
- name: Build llmgw-kb for every system
|
||||
run: |
|
||||
for target in windows/amd64 windows/arm64 darwin/amd64 darwin/arm64 linux/amd64 linux/arm64; do
|
||||
CGO_ENABLED=0 GOOS="${target%/*}" GOARCH="${target#*/}" go build -o /dev/null ./cmd/llmgw-kb
|
||||
done
|
||||
|
||||
# The Terraform provider is a module of its own, which ./... above does
|
||||
# not reach. Its tests check its client against docs/admin-api.openapi.json,
|
||||
# so a change to the admin API that breaks the provider fails here. The
|
||||
|
||||
@@ -55,6 +55,21 @@ jobs:
|
||||
(cd dist && sha256sum "${name}" >> checksums.txt)
|
||||
done
|
||||
|
||||
# llmgw-kb, which people run on their own machines to put their
|
||||
# documents into a knowledge base: Windows too, which llmgw itself
|
||||
# does not build for.
|
||||
for target in windows/amd64 windows/arm64 darwin/amd64 darwin/arm64 linux/amd64 linux/arm64; do
|
||||
os="${target%/*}"; arch="${target#*/}"
|
||||
ext=""; [ "$os" = windows ] && ext=".exe"
|
||||
name="llmgw-kb-${version}-${os}-${arch}${ext}"
|
||||
echo "building $name"
|
||||
CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build \
|
||||
-trimpath \
|
||||
-ldflags "-s -w -X gitea.mynodes.xyz/ilker/llmgw/internal/version.Version=${version}" \
|
||||
-o "dist/${name}" ./cmd/llmgw-kb
|
||||
(cd dist && sha256sum "${name}" >> checksums.txt)
|
||||
done
|
||||
|
||||
# The reverse proxy configurations, so a release page carries
|
||||
# everything install.md asks for and the repository is not needed.
|
||||
for f in deploy/proxy/Caddyfile deploy/proxy/nginx.conf; do
|
||||
@@ -91,6 +106,7 @@ jobs:
|
||||
# test the cross-compiled ones, which is why the native build is
|
||||
# checked rather than assumed.
|
||||
./dist/llmgw-${version}-linux-amd64 --version
|
||||
./dist/llmgw-kb-${version}-linux-amd64 --version
|
||||
cat dist/checksums.txt
|
||||
|
||||
# Signed, so a machine that trusts the key built into it installs
|
||||
|
||||
+626
-14
@@ -386,10 +386,9 @@ portal is not done — the page shows; the console and ctl change.
|
||||
tenant-admin / billing; `ctl audit`. Depends on A5 for a real actor.
|
||||
2. **Done (branch `features-c`):** OIDC sign-in (`auth.oidc`, PKCE,
|
||||
verified email, roles and memberships from groups, `no_passwords`) and
|
||||
SCIM 2.0 (`/scim/v2` Users and Groups). Left: the sign-in cookie is
|
||||
signed with a per-process key, so with two control planes behind a
|
||||
balancer the callback must reach the one that started it (sticky
|
||||
sessions) until the key comes from the store. **OIDC/SSO login (M), then SCIM (M).** `internal/account/service.go`,
|
||||
SCIM 2.0 (`/scim/v2` Users and Groups). The sign-in cookie's key comes
|
||||
from the store (2026-10-10), so a callback may reach either control
|
||||
plane behind a balancer. **OIDC/SSO login (M), then SCIM (M).** `internal/account/service.go`,
|
||||
`internal/web/account.go`; map IdP groups to org/team/admin.
|
||||
3. **Done (branch `features-c`):** key lifetime limits (gateway, org, team,
|
||||
project), `keys rotate --overlap`, kinds and expiry in `keys list`.
|
||||
@@ -405,8 +404,9 @@ portal is not done — the page shows; the console and ctl change.
|
||||
lost, budget spent, update available; signed JSON or Slack; retries,
|
||||
hourly de-duplication), `ctl notify test`, Alertmanager receiver
|
||||
templates. Left: `server init` and the console do not yet ask for a
|
||||
receiver — the templates are filled in by hand; each control plane sends
|
||||
budget events on its own, so two behind a balancer may both send one.
|
||||
receiver — the templates are filled in by hand. Events are claimed in
|
||||
the store before they are sent (2026-10-10), so two control planes send
|
||||
each once.
|
||||
**Alerting out of the box (S).** `deploy/monitoring/alertmanager.yml`
|
||||
ships `receiver: none`; add Slack/email/webhook/PagerDuty receiver
|
||||
templates driven by `server init` or the console; llmgw-native webhooks
|
||||
@@ -569,9 +569,12 @@ portal is not done — the page shows; the console and ctl change.
|
||||
client tests against a fake admin API and the OpenAPI document;
|
||||
acceptance tests with Terraform and OpenTofu against the fake; one
|
||||
apply, empty plan, in-place change and destroy against a real `llmgw
|
||||
server` run by hand. Left: organizations, teams, projects, people and
|
||||
keys are not in the admin API (they are `llmgw org`/`users`/`keys`
|
||||
against the store), so not in the provider; a model's definition beyond
|
||||
server` run by hand. Organizations, teams, projects, keys and
|
||||
memberships are in the admin API since 2026-10-08 (`/api/v1/orgs`,
|
||||
`teams`, `projects`, `keys`, `users/{id}/memberships`) and in the
|
||||
provider (`llmgw_org`, `llmgw_team`, `llmgw_project`, `llmgw_key`,
|
||||
`llmgw_membership`); accounts are still made by sign-up, invitation or
|
||||
OIDC/SCIM. Left: a model's definition beyond
|
||||
scaling, class, owner and adapters is changed by replacing it (the API
|
||||
has no edit, and `resources` is not used yet); `canary`, `slo`,
|
||||
`scaling.schedule` and `prewarm`, `guard` role and `origin` are not in
|
||||
@@ -793,10 +796,19 @@ leader 30s later with /health answering throughout; rollback did the same.
|
||||
facility state from the store (it kept what it read at start).
|
||||
- Retires `appendGuardIfMissing` (server init): the import does it.
|
||||
|
||||
### E2. Generated files from settings (L) — firewall on/off done 2026-10-06
|
||||
### E2. Generated files from settings (L) — firewall on/off done 2026-10-06, every machine 2026-10-09
|
||||
`firewall.enabled` in the cluster's settings reaches nodes over the agent
|
||||
stream and control planes through llmgw-firewall-sync.path. Left: the mirror
|
||||
machine and store members, the allowed set from membership, the other files.
|
||||
stream and control planes through llmgw-firewall-sync.path. Since
|
||||
2026-10-09 the mirror machine too (the leader PUTs /v1/firewall, the mirror
|
||||
writes `.firewall.json`, the same path unit applies it) and store members
|
||||
that run no llmgw (the leader keeps it in `cluster_state`, a root timer
|
||||
reads the member's own copy with psql); `firewall_networks` reach all of
|
||||
them, and the mirror lets in the network of every node and control plane
|
||||
the leader knows. Tried on the PoC: off and on, and a network added, on
|
||||
control planes, mirror, store member and nodes. Left: the allowed set
|
||||
replaced by membership rather than added to (a network taken out of the
|
||||
settings stays let in), the other files (the mirror's unit flags,
|
||||
keepalived, etcd and Patroni configuration).
|
||||
|
||||
Every machine runs the agent (root) with roles — node, control plane, mirror,
|
||||
store member — and the agent renders and applies what was written once by
|
||||
@@ -807,17 +819,617 @@ Patroni configuration. The firewall's allowed set is the cluster's membership
|
||||
removed changes every machine's rules. Retires `firewall.Admit` and
|
||||
`firewall_networks` in server.yaml (imported into the store).
|
||||
|
||||
### E3. agent.yaml down to bootstrap (M)
|
||||
### E3. agent.yaml down to bootstrap (M) — done 2026-10-08 (branch `settings-e`)
|
||||
agent.yaml keeps the control planes' addresses, its certificate and its id.
|
||||
Labels, fabrics, engines, heartbeat and supervisor settings move to the
|
||||
store per node and reach the agent over its stream; a change to the file is
|
||||
noticed and the agent restarts itself.
|
||||
Built as a section of the cluster's settings rather than a store of its own,
|
||||
so history, audit, `ctl config`, the console and the portal page cover it:
|
||||
`agents.default` and `agents.nodes.<id>` (config.AgentSettings; bootstrap
|
||||
keys node_id/server(s)/hostname/mgmt_addr/tls refused; each node's merge
|
||||
loaded as an agent would, so a bad fabric is refused on save). A live key:
|
||||
the control plane sends ControlMessage.AgentSettings on register and, a
|
||||
second apart per node, on change; the agent compares with what it started on,
|
||||
checks it loads beside its agent.yaml (else logs and keeps running), keeps it
|
||||
in /var/lib/llmgw-agent/agent-settings.yaml and restarts (engines adopted).
|
||||
At start: agent.yaml < kept settings < LLMGW_ env; kept settings that no
|
||||
longer load are set aside, never a reason not to start. agent.yaml is
|
||||
watched every 10 s and restarted on when it changes and loads. Left: file
|
||||
keys are not imported into the settings (they keep working under them);
|
||||
`node add` still writes a full agent.yaml; labels have two homes now
|
||||
(`ctl label`, which wins, and agents.*.labels).
|
||||
|
||||
### E4. Web settings page (S)
|
||||
### E4. Web settings page (S) — done 2026-10-08 (branch `settings-e`)
|
||||
The same document, edited on /admin/settings, for whoever has no terminal.
|
||||
Built: administrators only (staff roles get 404: webhook secrets), checked by
|
||||
the same function as PUT /api/v1/settings, made from a base version or
|
||||
refused on a newer one (the edit kept in the editor), history with view and
|
||||
roll back as a new version, audit with the size not the body, English and
|
||||
Turkish, a Settings link in the menu.
|
||||
|
||||
---
|
||||
|
||||
## F. Company documents: knowledge bases (RAG), then adapters (2026-10-08)
|
||||
|
||||
A company wants a model to answer from its own documents. Retrieval is the
|
||||
tool for *knowledge* (sources shown, re-indexed in minutes, access per
|
||||
tenant); LoRA training is for *behaviour* (vocabulary, format) and has no
|
||||
access control — whatever goes into weights reaches everyone who may use the
|
||||
adapter. RAG first. Decisions taken: access per collection (no per-document
|
||||
ACL yet); a personal key or session names a project or team and gets that
|
||||
chain only, never the union of its person's memberships; PostgreSQL store
|
||||
only, off on SQLite with a sentence saying why.
|
||||
|
||||
### F1. Store: collections, grants, chunks (M) — done 2026-10-08 (branch `rag`)
|
||||
Built (`internal/store/knowledge.go`, migration 0035 PostgreSQL only):
|
||||
- `kb_collections` (owner scope/id, org, embed_model, dims), `kb_grants`
|
||||
(read|write, any scope, never across organizations; a key needs a
|
||||
project; a person needs a membership in the organization),
|
||||
`kb_documents` (digest: the same again is no change, a new one replaces
|
||||
in one transaction).
|
||||
- `kb_chunks` is created when the store opens, not in a migration (a
|
||||
migration needing an extension would stop the control plane; one that
|
||||
skipped it would never run again): partitioned by organization (hashed
|
||||
partition name), `tsv` generated (`simple`), GIN on the parent, one
|
||||
partial HNSW index per collection on its organization's partition
|
||||
(`embedding::vector(dims)`, `WHERE collection = '<id>'`). EXPLAIN on
|
||||
20k rows: partition pruned, that index used, policy applied.
|
||||
Later (scale, 2026-10-08): a partial GIN per collection too
|
||||
(`kb_tsv_<id>`, backfilled at open); `vec_type` per collection, `halfvec`
|
||||
on pgvector ≥ 0.7 (up to 4000 dims); `SET LOCAL hnsw.ef_search` to
|
||||
candidates(k) clamped 40..1000 (EXPLAIN: 40 rows without it, all with
|
||||
it); the text arm takes candidates by `ts_rank` in a subquery and scores
|
||||
only those.
|
||||
- RLS FORCEd on `kb_chunks` against `llmgw.kb_org`/`llmgw.kb_collections`
|
||||
(set with set_config per transaction), one role — the table owner — not
|
||||
a separate one. Chunks are deleted by the foreign-key cascade from their
|
||||
document, which RLS does not block.
|
||||
- `CollectionsFor(Effective, write)`: owner or grant anywhere on the chain;
|
||||
no organization (personal key without a project) reads nothing.
|
||||
- `SearchChunks`: per collection nearest-neighbour (literal id and cast so
|
||||
the partial index is used) plus a full-text arm with any word
|
||||
(`a | b`, lowercased by the database, not Go), fused by reciprocal rank.
|
||||
- Deleting a project/team/org that owns collections is refused like one
|
||||
with members (changed from "deleted with them": every other delete in the
|
||||
tree refuses); DeleteUser/ForgetUser delete the person's collections;
|
||||
deleting an organization drops its partition; grants to a deleted scope go.
|
||||
- `server init` installs `postgresql-N-pgvector` / `pgvector_N` and enables
|
||||
it; pgha installs it on every member at setup and enables it on the
|
||||
leader on every run. Neither stops on failure.
|
||||
Left: a cluster set up before (the PoC) gets no package from a rerun of
|
||||
pgha on a running cluster — install by hand or add an upgrade step;
|
||||
`server restore`/`CopyTo` do not carry kb tables (SQLite → PostgreSQL only,
|
||||
and SQLite has none); tests need `LLMGW_TEST_POSTGRES` pointing at a
|
||||
pgvector database as a non-superuser (a superuser bypasses RLS).
|
||||
|
||||
### F2. Ingest on the user's machine (L) — done 2026-10-08 (branch `rag`)
|
||||
Uploading whole files to the control plane was dropped: a PDF is 20–100×
|
||||
its text, and the work belongs where the files are. Instead a small client,
|
||||
`llmgw-kb`, run by the person on their own machine with their own key:
|
||||
- **Its own binary** (`cmd/llmgw-kb`), for Windows, macOS and Linux on amd64
|
||||
and arm64. `llmgw` itself does not build on Windows (agent syscalls), and
|
||||
employees' laptops should not carry the server, agent and setup screens. A
|
||||
test keeps server packages out of its imports.
|
||||
- `llmgw-kb sync <collection> <dir|file…> [--include/--exclude globs]
|
||||
[--dry-run] [--delete]`: scan → local catalog (path, size, mtime, sha256;
|
||||
cache under os.UserCacheDir, digests not recomputed for unchanged files)
|
||||
→ the collection's manifest from the gateway (name → digest only) → diff
|
||||
→ for new/changed files only: extract, chunk, embed through the gateway's
|
||||
`/v1/embeddings` with the person's key (its accounting, limits, guard and
|
||||
band apply), and send name, digest, chunk texts and vectors; `--delete`
|
||||
removes documents whose files are gone. `--dry-run` lists what would leave
|
||||
the machine and how much. A file that cannot be read (locked, unsupported,
|
||||
no text) is skipped with the reason; the run goes on.
|
||||
- **Formats, as many as possible.** Built in, pure Go: PDF (text layer),
|
||||
DOCX/XLSX/PPTX, ODT/ODS/ODP, RTF, EPUB, HTML/XHTML, EML (text and HTML
|
||||
parts), Jupyter notebooks, subtitles (SRT/VTT), CSV/TSV, JSON/YAML/XML,
|
||||
Markdown/RST/AsciiDoc/Org, plain text and source code; ZIP archives
|
||||
recursed into. With a helper found on the machine: legacy DOC/XLS/PPT, RTF
|
||||
and anything else LibreOffice opens (`soffice --headless --convert-to`),
|
||||
images and scanned PDFs through Tesseract (`tur+eng`). Text in UTF-8,
|
||||
UTF-16 (BOM) or Windows-1254/1252 is converted to UTF-8.
|
||||
- **Across systems:** document names relative to the synced root, `/`
|
||||
separated, Unicode NFC (macOS hands out NFD), a warning for two names that
|
||||
differ only in case; CRLF → LF; key from `LLMGW_API_KEY` or `llmgw-kb
|
||||
login` (os.UserConfigDir, 0600 where that means something); the system's
|
||||
CA store, `--ca` for another.
|
||||
- **Gateway endpoints** (client listener, person's key):
|
||||
`GET /v1/kb/collections` (what this key may read/write),
|
||||
`GET /v1/kb/collections/{id}/manifest`, `PUT
|
||||
/v1/kb/collections/{id}/documents` (one document per request: name,
|
||||
digest, bytes, chunks with text and embedding; dims checked, chunk count
|
||||
and size bounded), `DELETE …/documents?name=`. Write needs a write grant
|
||||
or ownership on the key's chain (`CollectionsFor(…, write)`); otherwise
|
||||
404.
|
||||
- **Administration** stays with operators: `llmgw ctl kb
|
||||
create|list|delete|grant|revoke|docs`, admin API `/api/v1/kb`, audit,
|
||||
console screen; `docs/knowledge.md`.
|
||||
- Trade-off accepted: vectors come from the client, so a writer could store
|
||||
vectors that do not match their text — only in collections they may
|
||||
already write.
|
||||
Built: `internal/kb/extract` (formats above; helpers found on PATH and in
|
||||
each system's install places), `internal/kb/chunk` (≈1800 characters under
|
||||
the heading path, a paragraph of overlap, sections kept apart once a chunk
|
||||
is a third full), `internal/kb/kbsync` (ledger in SQLite at
|
||||
os.UserConfigDir/llmgw-kb/ledger.db: documents keyed by gateway, collection
|
||||
and name with file, size, mtime, digest, state, reason; runs), `cmd/llmgw-kb`
|
||||
(login, collections, sync, status, history, formats), `internal/router/kb.go`
|
||||
(the four gateway endpoints, `X-Llmgw-Project` for personal keys, audit),
|
||||
`internal/control/adminapi/kb.go` + `llmgw ctl kb` + console Knowledge
|
||||
screen, `docs/knowledge.md`; CI builds the client for all six targets and
|
||||
releases attach it. Tested: every format from generated fixtures and a
|
||||
Turkish PDF (OCR path with Tesseract here), sync against a fake gateway and
|
||||
end to end against the router on pgvector.
|
||||
Tried on the PoC 2026-10-08 (control planes 140 and 150 on a dev build of
|
||||
this branch; their previous binary is /usr/local/bin/llmgw.pre-rag): Ubuntu's
|
||||
pgvector 0.6.0 installed on 149/150 and enabled; bge-m3's weights were never
|
||||
in the mirror (fetched from CompendiumLabs/bge-m3-gguf@1864f47f) and it is
|
||||
held at 1 replica; tree Demo/ik/belgeler and Demo/satis/crm with a key each;
|
||||
collection Demo/politikalar filled from a Mac through an SSH tunnel with ten
|
||||
Turkish files in nine formats (9 sent in 5 s, the .doc skipped for want of
|
||||
LibreOffice; a rerun sent nothing in 0 s; a changed file alone was sent).
|
||||
Nine Turkish questions each found their page first once search scored by
|
||||
similarity plus word coverage (rank fusion had got two wrong). The other
|
||||
team's key saw no collection and got 404 by id; with a read grant it read
|
||||
and could not write; revoked, nothing again. On the live store the llmgw
|
||||
role (not superuser, no BYPASSRLS) sees 0 chunks with no scope set and 0
|
||||
with the organization alone.
|
||||
Left: the gateway's collection list shows the owner as team:ID, not its
|
||||
path; no LibreOffice on the dev machine, so the legacy
|
||||
Office path is untested beyond its error; the client is unsigned (Gatekeeper
|
||||
and SmartScreen warn); Outlook `.msg` is not read; one document over ~60 MiB
|
||||
of JSON (a few thousand pages) is refused rather than split.
|
||||
PostgreSQL tests of store, router and adminapi share LLMGW_TEST_POSTGRES and
|
||||
the store's TRUNCATE each test; run them with `go test -p 1` or one package
|
||||
at a time, or a router test loses its rows mid-run.
|
||||
|
||||
### F3. Retrieval on the request path (M) — done 2026-10-08 (branch `rag`)
|
||||
- `llmgw.knowledge: [collections]` in the body (stripped before the
|
||||
engine), or default collections per project/key.
|
||||
- Allowed set from the caller's chain (§7.9.2), cached like limits; grants
|
||||
add up down the tree. Personal keys and sessions need `x-llmgw-project`
|
||||
(or the playground's picker) naming a scope the person is a member of.
|
||||
- A collection not allowed answers 404, exactly as a missing one.
|
||||
- Hybrid search with the filter inside the query (per collection, its own
|
||||
partial index; pgvector 0.6 is enough —
|
||||
scored by similarity plus the share of the question's words held (built
|
||||
in F1/F2 as SearchChunks), `/v1/rerank` when a reranker is in the catalog. Top
|
||||
chunks into a system message with document and position;
|
||||
`x-llmgw-sources` lists them.
|
||||
- Leaks closed: answer-cache key includes the allowed set; per-organization
|
||||
`cache_salt` on RAG requests; injected passages under `no_store` and the
|
||||
guard's prompt-injection check; returned doc ids in the audit.
|
||||
- Done: router tests that tenant B never receives tenant A's chunks
|
||||
(including via cache), that a personal key without a project is refused,
|
||||
metrics (retrieval latency, chunks returned), docs.
|
||||
|
||||
Built (`internal/router/rag.go`): `X-Llmgw-Knowledge` (header, as the
|
||||
other llmgw extensions are, not a body field), `X-Llmgw-Knowledge-K`;
|
||||
collections from the key's chain (kbEffective, shared with the kb
|
||||
endpoints), 404 for one not readable; the question embedded through the
|
||||
ordinary path on the caller's principal; SearchChunks; passages trimmed to
|
||||
the model's context; added to the one leading system message (or a new
|
||||
one); `x-llmgw-sources`; before guardInput; never cached; metrics. Default collections built (kb_defaults on org/team/project/key, narrowest
|
||||
wins, 30 s cache, `ctl kb default`, console action; never refuse); the
|
||||
portal chat's Documents picker (collection, default or none; sources listed
|
||||
under the answer; a header the page sends itself is dropped). `cache_salt` per organization on requests given passages, sent to vLLM
|
||||
replicas only (chosen replica's engine). Checked on the PoC 2026-10-08:
|
||||
vLLM 0.30.0 (cuda01) with TinyLlama-1.1B AWQ run by hand beside the 35B in
|
||||
the card's last 2.3 GB — a 668-token prompt reused 656 tokens under the same
|
||||
salt and 0 under another's, with identical text (llmgw cannot place it
|
||||
there: placement takes whole GPUs). Live on the PoC: a team default answers
|
||||
a request without the header, `none` turns it off, an explicit header with
|
||||
k=2 gives two sources and no more, llmgw-kb lists owners by path. Left: returned passages
|
||||
are not audited per request (the header and the kept request record them).
|
||||
|
||||
### F4. Adapter training (L, after F1–F3) — done 2026-10-08 (branch `adapters-f4`)
|
||||
A run-to-completion job placed by the control plane like a replica and
|
||||
supervised by the agent like an engine, in a Python environment the agent
|
||||
installs (TRL + PEFT, QLoRA). Data: Q/A pairs generated from a collection by
|
||||
a cluster model through `/v1/batches`, reviewed by people. Output to the
|
||||
mirror, into the model's `adapters`, through canary and a quality suite.
|
||||
Runs when the scaling schedule frees a GPU; 8–14B dense bases on one card.
|
||||
Never train on a collection some users of the adapter may not read.
|
||||
Tried 2026-10-08 on the PoC, by hand: Qwen2.5-0.5B-Instruct (fp16) with a
|
||||
LoRA (r=16, every linear layer, 8.8M trainable), trained beside the 35B in
|
||||
cuda01's last 2.3 GB (peak 2158 MiB on the card, 64 s for 138 steps, PEFT in a
|
||||
venv over the vLLM engine's torch). Data: 90 Turkish Q/A pairs a cluster
|
||||
model wrote from the nine Demo/politikalar documents, through the gateway.
|
||||
On nine questions it had not seen, phrased otherwise: base 0/9, 6 epochs
|
||||
4/9, 4 epochs at a lower rate 2/9. Wrong answers mixed facts across
|
||||
documents (the meal card's amount, the office day). The same nine through
|
||||
retrieval: 9/9 with sources. So F4 is for format, vocabulary and tone, and
|
||||
facts come from F1–F3; a bigger base on a free card would do better but not
|
||||
change that. The training itself fits a small slot on a shared card, which
|
||||
the job design can rely on for small bases.
|
||||
|
||||
Built: `ctl kb dataset` (POST /api/v1/kb/collections/{id}/dataset, NDJSON
|
||||
with each pair's collection); train_jobs/train_artifacts (migrations 0037,
|
||||
0038 owner); internal/control/train (queue, one job per node, artifacts in
|
||||
1 MiB pieces, a node that goes fails its job); internal/agent/train (the
|
||||
newest vLLM's python3 + `uv pip install --no-deps --target` peft 0.21.2 and
|
||||
accelerate 1.15.0; llama.cpp's conversion code for the installed release,
|
||||
untarred from its source; the card with the most free memory, if enough;
|
||||
runs as llmgw-engine); `ctl train start|list|show --follow|cancel|rm|
|
||||
download|publish`; the mirror's PUT /v1/models/{model}/{version}/{file};
|
||||
publish uploads the GGUF (llama.cpp) or PEFT files (vLLM) and sets the
|
||||
adapter after checkAdapters. Owners: catalog.Adapter.Owner, enforced in the
|
||||
router (404, hidden from /v1/models); a job's owner defaults to its single
|
||||
collection's owner, must be able to read every collection the pairs came
|
||||
from, and pairs from a knowledge base are never public. Tried on the PoC:
|
||||
dataset of 90 pairs in 89 s; a job trained beside the 35B (peak 2214 MiB,
|
||||
138 steps in about a minute) and sent back PEFT and GGUF.
|
||||
Left: a job dies with its agent's stream (no resume); no console screen for
|
||||
training yet; quality gating of a published adapter is by hand (ctl eval).
|
||||
|
||||
---
|
||||
|
||||
### F5. Large imports (M) — measured 2026-10-09, open
|
||||
The metallurgy corpus (2,314 files, 108,779 chunks, bge-m3 on the one
|
||||
GPU) went in at about 25 chunks/s, the same over a phone hotspot and the
|
||||
LAN. The engine's slots were the limit: llama.cpp `--parallel 4` embeds
|
||||
four texts per pass, while llmgw-kb sends 32 a request from 6 workers. The
|
||||
usage rows show 4.6 s a request on average, 1.8 s of it in the gateway's
|
||||
queue and 0.04 s in the guard. The GPU sat at 18–50% and the store machine
|
||||
at a third of its CPUs.
|
||||
- An embedding model's profile with many slots by default (`--parallel`
|
||||
32–64, `--ctx-size` to match, `--batch-size`/`--ubatch-size` at least the
|
||||
longest chunk), or `ctl model serve` choosing that for a model with
|
||||
`capabilities.embeddings`. With the default 512, one 516-token chunk failed.
|
||||
- Embedding on the gateway's side of a document PUT, so the client sends
|
||||
text only: today an embeddings answer is ~21 KB of JSON a chunk, and
|
||||
the text and vector go up again (~11 KB).
|
||||
- The store next: HNSW inserts took half of one of pg01's two cores; at a
|
||||
million chunks the vectors alone outgrow its 4 GB. halfvec (pgvector ≥
|
||||
0.7) and a larger store machine.
|
||||
- Done: a batch of embeddings checked text by text against the context
|
||||
(branch `embeddings-batch`); it was refused as one over-long prompt.
|
||||
- Done: `query_prefix` on an embedding model, before questions only.
|
||||
- Qwen3-Embedding-4B on the RTX 6000 (2026-10-09): llama.cpp Q8 1,575
|
||||
tok/s as first set up (32k cache, 16 slots), 2,700 with a 16k cache,
|
||||
FlashAttention and ubatch 1–2k; f16 no better. vLLM fp16 (`--runner
|
||||
pooling`) 6,200 direct, 4,700–4,900 through the gateway in the import,
|
||||
and its vectors match llama.cpp's (cosine 0.9995). The 103,579 chunks
|
||||
took 3 h. Done (2026-10-10, branch `admission-bulk`): a source per
|
||||
profile, so an embedding model's GPU profile runs vLLM and its CPU
|
||||
profile llama.cpp; a band's own `queue_ttl` for bulk imports, and
|
||||
`Retry-After` on a busy refusal, which is what had turned passages away
|
||||
(llmgw-kb retries only what says when to).
|
||||
- Retrieval on 40 questions written from the corpus, document in the top
|
||||
5: bge-m3 31/40 (Turkish questions 13/21), Qwen3-Embedding-4B 37/40
|
||||
(18/21), with its query prefix 36/40 and first place 31/40 instead of
|
||||
28 (MRR 0.738 → 0.789 → 0.838). Next: translating the question with the
|
||||
chat model, and a reranker.
|
||||
|
||||
## G. Jobs: 3D generation through ComfyUI (2026-10-10)
|
||||
|
||||
A request for a 3D model takes minutes on the one 24 GB card, and the card has to be
|
||||
emptied of the chat model first. That is not a request to hold a socket for
|
||||
— `internal/router/activate.go:46` already says so — it is a job: queued,
|
||||
answered with an id, asked after, its result downloaded. The job machinery
|
||||
is built first and tested without a GPU; the ComfyUI generalisation and the
|
||||
custom-node install come after. Decided 2026-10-10: a `jobs` table of its
|
||||
own rather than a one-line batch (the JSONL shape would be artificial and
|
||||
has nowhere for progress), and progress is shown, not just a status.
|
||||
|
||||
Which models (decided 2026-10-10): the assets go into a game sold on
|
||||
Steam, worldwide, the EU included, so the licence of the output decides
|
||||
before the quality does. Hunyuan3D (every version) is out: Tencent's
|
||||
licence excludes the EU, UK and South Korea, and whether a game shipped
|
||||
there with its outputs is "use in the Territory" is not a question to
|
||||
leave to a court. In, in this order: **TRELLIS.2** shape (MIT; the best
|
||||
geometry; its own texture baking goes through nvdiffrec and nvdiffrast,
|
||||
both under NVIDIA's non-commercial source licence, so that stage is not
|
||||
run), **Step1X-3D** (Apache 2.0, StepFun; shape and texture in one, the
|
||||
EU-safe textured path — its ComfyUI node and VRAM need are to be
|
||||
checked), **Direct3D-S2** (MIT, DreamTech; high-detail shape, has a
|
||||
ComfyUI node) if TRELLIS.2's detail falls short. Not in: TripoSR and
|
||||
InstantMesh (permissive but superseded), Stable Fast 3D (Stability's
|
||||
community licence, revenue-capped; served all the same since 2026-10-10
|
||||
at the user's request, see G3), SAM 3D Objects (Meta's own licence,
|
||||
to be read before it is considered). Steam's disclosure form (changed
|
||||
2026-01-16) lists pre-generated 3D models as content that must be
|
||||
declared on the store page: the licence notice G3 writes into the GLB is
|
||||
what the studio answers that form from.
|
||||
|
||||
### G1. Jobs: store, worker, endpoints (M) — done 2026-10-10 (branch `jobs-g1`)
|
||||
Built as written, with two choices: a job found running after its gateway
|
||||
died is run again from the start (the prompt id column is there for G2 to
|
||||
follow the same work instead), and the request is validated at the door
|
||||
only as far as costs nothing — key, model, allowlist, the request's shape
|
||||
— with limits, guardrails and admission applied when the worker replays it,
|
||||
said as the job's error. The result file's `content_type` is the job's; the
|
||||
files endpoint still serves every file as an octet stream.
|
||||
- Store: `jobs` (id, owner key/principal, kind, model, status, input
|
||||
(JSON), progress (JSON: queue position, percent, step), result file id,
|
||||
error, created/started/finished, lease owner/until), migrations for
|
||||
sqlite and postgres beside `internal/store/batches.go`, whose
|
||||
claim/renew/release lease pattern is copied, not shared: a job is one
|
||||
unit of work, a batch is many lines.
|
||||
- Worker: `internal/router/jobworker.go` after `batchworker.go`. Any
|
||||
gateway on the store claims a queued job, asks for the model with
|
||||
`activator.Want` and waits as long as it takes (no `ActivationWait`
|
||||
bound: nobody is holding a connection), runs it, stores the output
|
||||
through the files store, writes usage and price, marks it done. A
|
||||
gateway that dies mid-job loses its lease and another one picks it up;
|
||||
the ComfyUI prompt id is in the job row so the second gateway asks
|
||||
after the same work rather than queueing it again.
|
||||
- Endpoints: `POST /v1/3d/generations` validates like any inference
|
||||
request (key, allowlist, limits, admission, guardrails on the prompt),
|
||||
writes the job, answers 202 with `{id, status: "queued"}`;
|
||||
`GET /v1/3d/generations/{id}` returns status, progress and, when done,
|
||||
`output_file_id` and the usage; `POST .../{id}/cancel`; the file comes
|
||||
from `GET /v1/files/{id}/content`. Owner-scoped like batches. Results
|
||||
are kept for the batch files' retention and deleted with them: a
|
||||
departure from `images.go`'s "kept nowhere", because a mesh is a
|
||||
download, not a JSON field; the retention is the bound.
|
||||
- Done when: a job survives the gateway that took it being killed, a
|
||||
cancelled job leaves ComfyUI's queue (`comfy.Client.cancel`), two
|
||||
gateways never both run one job (test with the lease), and the usage
|
||||
row carries the job's price.
|
||||
|
||||
### G2. Progress from ComfyUI (S, after G1) — done 2026-10-10 (branch `jobs-3d`)
|
||||
Built as written: the gate's websocket under its own client id, the
|
||||
client_id of every /prompt rewritten, /progress/{prompt_id} on the gate,
|
||||
the router reading it and /queue into the job (only into a job of the same
|
||||
key). Left: a node that reports no steps (TripoSR's) shows its name and no
|
||||
percent; a vanished replica is said by the job failing, not by progress.
|
||||
The gate (`internal/comfy/gate.go`) passes seven paths and no websocket.
|
||||
Rather than open `/ws`, the gate subscribes to ComfyUI's own `/ws` once,
|
||||
with `golang.org/x/net/websocket` (already a dependency), under a client
|
||||
id of its own, and rewrites the `client_id` of every `/prompt` it
|
||||
forwards to that one. It keeps the last `executing`, `progress`
|
||||
(value/max/node) and `execution_error` message per prompt id and serves
|
||||
them at `GET /progress/{prompt_id}` on the gate; the router polls that
|
||||
beside `/history` and `/queue` and writes queue position, percent and the
|
||||
current node's class into the job's progress. The gate's buffer is
|
||||
bounded (the last N prompts) and the websocket reconnects. Done when a
|
||||
3D job shows "queued, 2 ahead" then "running 40%, step 12/30" in
|
||||
`ctl jobs` and on the account page, and a job whose replica vanished
|
||||
says so within one poll.
|
||||
|
||||
### G3. ComfyUI beyond pictures: 3D models and custom nodes (M, after G1) — done 2026-10-10 (branch `jobs-3d`), with TripoSR
|
||||
Built: output kinds, image upload through the gate, every output list
|
||||
fetched, OBJ to GLB, licence in /v1/models, the job and the GLB, per_mesh
|
||||
and usage.meshes (migration 0045), custom nodes cloned at their commit by
|
||||
the gate with packages beside the replica (a node's named packages at its
|
||||
range, the rest at ComfyUI's versions). Not as written: the model. On the
|
||||
PoC's Turing card TRELLIS.2 cannot run — it needs Meta's gated DINOv3,
|
||||
its published background remover (RMBG-2.0) is non-commercial, and its
|
||||
sparse attention and Direct3D-S2's need flash-attention (Ampere and
|
||||
newer). TripoSR (MIT) proves the chain on cuda01: a .glb from a picture
|
||||
in about 30 s. Left: background removal before TripoSR (BiRefNet, MIT,
|
||||
as a node), TRELLIS.2 or Direct3D-S2 on an Ampere card with an ungated
|
||||
encoder, a missing node refusing the replica by name.
|
||||
Added 2026-10-10 at the user's request, beside the order above: **Stable
|
||||
Fast 3D** as `sf3d` (Stability AI Community License: free under US$1 M
|
||||
annual revenue, an enterprise licence above, "Powered by Stability AI"),
|
||||
the textured mesh that runs on Turing: about 1 s a mesh warm in fp16,
|
||||
UV-mapped GLB with its texture, 18 GB of the card at peak. Its weights are
|
||||
gated, so they were fetched once with the user's token into
|
||||
/srv/llmgw-comfy/stable-fast-3d and linked into the engine user's
|
||||
Hugging Face cache, which its loader node reads by repository name; no
|
||||
token is on the machine at runtime. Its own save node returns the GLB as
|
||||
base64, which the gate cannot fetch, so `LLMGWSaveMesh` writes the file;
|
||||
`LLMGWPrepareForMesh` now also gives the object's mask. Left: its two
|
||||
CUDA extensions (texture_baker, uv_unwrapper) are not on PyPI, so they
|
||||
were built on cuda01 and installed into the engine's venv by hand; a
|
||||
custom node's requirements need a way to name a local build, or the
|
||||
engine install must carry them, before a customer site gets this. And
|
||||
rembg's default session now fetches BRIA RMBG-2.0 (non-commercial):
|
||||
every workflow must name its rembg model (ours say u2net).
|
||||
- Catalog: a model makes `pictures` or `meshes` (`MakesImages` becomes an
|
||||
output kind; `ServesComfyUI` stays). 3D weight folders in the gate's
|
||||
`modelFolders`; the workflow's placeholders gain `image` (the input
|
||||
picture, base64 in the request, written to the replica's `input/` by
|
||||
the gate through a new `POST /upload/image` the gate allows — or a
|
||||
base64-loading node, whichever TRELLIS.2's wrapper wants).
|
||||
- `comfy.Client.fetch` reads every output key, not `images` only, and
|
||||
fetches each file named by `/view`; `Image` becomes `Artifact` with its
|
||||
content type (model/gltf-binary, application/octet-stream for .obj).
|
||||
- Custom nodes: a model (or the engine config) names git repositories
|
||||
and a pip requirements list; the agent's provision step clones them
|
||||
into the replica's `custom_nodes/` (the gate's `--base-directory` is
|
||||
where ComfyUI looks, not the install directory) and installs the
|
||||
requirements into the engine's venv with the same PyTorch index and
|
||||
lock discipline as A8. A replica with a missing node refuses to start
|
||||
with the node's name in the error rather than ComfyUI's traceback.
|
||||
- Licence of the output (decided 2026-10-10: it matters). A catalog model
|
||||
carries `license: {name, url, notice}`; a job's answer, `/v1/models`
|
||||
and the portal page show it beside the result, and the mesh's GLB
|
||||
`asset.extras` carries the notice too, since the file travels without
|
||||
the API. What was checked on 2026-10-10: TRELLIS.2 weights are MIT and
|
||||
its outputs free, but its texture baking depends on nvdiffrast, whose
|
||||
NVIDIA Source Code License is non-commercial — a commercial service
|
||||
must render textures another way or skip that stage. Hunyuan3D 2.0 and
|
||||
2.1 claim no rights in outputs, but the licence excludes the EU, UK and
|
||||
South Korea, needs Tencent's permission above 1 M monthly active users,
|
||||
forbids using outputs to train other models, and asks for "Powered by
|
||||
Tencent Hunyuan"; the notice ships with any distribution. TripoSR is
|
||||
MIT, InstantMesh Apache 2.0. Every one of them trained on Objaverse,
|
||||
some of whose objects are CC-BY-NC: a provenance question no licence
|
||||
text settles, to be said in the notice rather than hidden.
|
||||
- Pricing: `per_image` stays; `per_mesh` beside it in `accounting.Price`,
|
||||
`Outcome.Meshes`, the usage sums and exports (as eaea671 did for
|
||||
pictures).
|
||||
- Done when: TRELLIS.2 on cuda01 with `scaling: {min: 0,
|
||||
idle_timeout: 10m}` makes a .glb from a picture through G1's endpoint
|
||||
with nvdiffrast and nvdiffrec not installed in its venv (the licence
|
||||
is kept by the install, not by a note), the GLB carries the MIT
|
||||
notice,
|
||||
the card is free again after the idle timeout, and the same request
|
||||
against flux-schnell is refused as "makes pictures, not meshes".
|
||||
|
||||
### G4. Portal and console (S, after G2) — done 2026-10-10 (branch `jobs-3d`)
|
||||
Built: /3d with model-viewer served from the gateway, progress, download
|
||||
and licence; `ctl 3d list|get|cancel` over /api/v1/client-jobs (ctl jobs
|
||||
was taken by the control plane's own work). Left: the console's Jobs view.
|
||||
A 3D page beside `/images` (upload a picture, watch progress, download
|
||||
the .glb, a viewer with `<model-viewer>` from the allowed CDNs), `ctl
|
||||
jobs list|get|cancel`, the console's Jobs view. The page sends down the
|
||||
same path as the API, like `imagesSend` does.
|
||||
|
||||
## H. ComfyUI as a platform (written 2026-10-11)
|
||||
|
||||
G made one model at a time work: flux for pictures, TripoSR and SF3D for
|
||||
meshes, each with its own endpoint, its own request shape and its own
|
||||
replica. The user's direction (2026-10-11): llmgw is a generic platform,
|
||||
and should serve whatever ComfyUI can run — pictures, edits, video,
|
||||
audio, meshes, upscaling, segmentation — as data in the catalog rather
|
||||
than code per model. Two decisions shape everything below:
|
||||
|
||||
- **The licence is metadata, not a filter.** llmgw does not leave a
|
||||
model out because of its licence. The licence travels with the model
|
||||
and with every file it makes; what a site will not serve is that
|
||||
site's policy (H6). The game studio's EU constraint is one such policy,
|
||||
not a decision the code makes for everyone.
|
||||
- **The install fits the hardware.** The PoC has one Turing card; other
|
||||
sites will have Ampere, Ada and Hopper. A catalog entry must be
|
||||
writable once and placed sensibly anywhere: fp16 on Turing, bf16 or
|
||||
fp8 where the card has them, flash-attention models only where it
|
||||
runs, CUDA extensions built for the card that runs them (H2, H4).
|
||||
|
||||
The first set of models proves the structure at its corners rather than
|
||||
by licence: Qwen-Image-Edit (several weight files, two pictures in),
|
||||
Wan 2.2 small (video out, a long job), ACE-Step (audio out), SF3D (a
|
||||
package not on PyPI, gated weights), Hunyuan3D (a licence that excludes
|
||||
the EU, which H6 must be able to say no to).
|
||||
|
||||
### H1. A workflow says what it takes and makes (M) — built 2026-10-11, not yet tried on the PoC
|
||||
Built as written, with three choices: a run answers with one file, the
|
||||
first of its output kind (several outputs as one zip is left); the
|
||||
workflow's placeholders are checked against the inputs when a run is
|
||||
served, since the catalog never reads the workflow file; and video,
|
||||
audio and files count as nothing in usage until a per-run price exists
|
||||
(per_image and per_mesh still count pictures and meshes).
|
||||
- Catalog: `comfyui.inputs` — a list of `{name, type, required, default,
|
||||
min, max, choices, max_mb, doc}`, type one of text, int, float, bool,
|
||||
choice, image, mask, audio, video, file. Each is a placeholder of the
|
||||
same name in the workflow. The template's own placeholders are checked
|
||||
against the list when the catalog is built: one the workflow uses that
|
||||
no input or built-in names is refused there, not at the first request.
|
||||
- `comfyui.output` gains `video`, `audio` and `files` beside `pictures`
|
||||
and `meshes`; `files` takes whatever the workflow saves. `comfy.Image`
|
||||
becomes `comfy.File` and `typeByName` knows mp4, webm, gif, wav, flac,
|
||||
mp3, ogg.
|
||||
- Endpoint: `POST /v1/workflows/runs` with `{model, inputs: {...}}`, a
|
||||
job like G1's (`GET /v1/workflows/runs[/{id}]`, `.../cancel`). A file
|
||||
input is base64, a data URL, or a `file-…` id from `/v1/files` for
|
||||
what is too big to send inline (video, long audio). Files go into the
|
||||
replica's input directory through the gate's `/upload/image`, which
|
||||
ComfyUI uses for any file. `/v1/images/generations` and
|
||||
`/v1/3d/generations` stay as they are, as fixed shapes over the same
|
||||
machinery.
|
||||
- The models list (`/v1/models`) gives each ComfyUI model its inputs and
|
||||
output kind, so a client can build a form without reading the catalog.
|
||||
- Done when: the existing flux, triposr and sf3d entries work unchanged,
|
||||
and one model with a picture and a text input (Qwen-Image-Edit) runs
|
||||
through `/v1/workflows/runs` with a 400 for a missing required input
|
||||
before any job is written.
|
||||
|
||||
### H2. Profiles that fit the hardware (M) — built 2026-10-11
|
||||
Built as written. A profile by need is left out of placement until the
|
||||
cluster's cards are known: with no class, placement would have read it as
|
||||
a CPU profile. A profile's own `source` is loaded under its own file name
|
||||
by the workflow's `{{checkpoint}}`.
|
||||
- A profile may say `gpu: {min_compute_capability, min_memory_gb}` in
|
||||
place of `gpu_class`. When the reconciler places a model, such a
|
||||
profile stands for every GPU class in the cluster that meets it, at
|
||||
the profile's priority: the catalog function that turns profiles into
|
||||
placement's view takes the cluster's classes (class, compute
|
||||
capability, memory) and returns one concrete profile per class that
|
||||
fits, named `<profile>@<class>`. Placement itself is unchanged: it
|
||||
still sees exact classes, and a tensor-parallel group still never
|
||||
mixes them.
|
||||
- A ComfyUI profile may carry its own `comfyui` block (workflow, nodes,
|
||||
checkpoint) over the model's: the fp8 workflow on compute 8.9 and up,
|
||||
the fp16 one on 7.5. A Turing card is not an error, it is the profile
|
||||
that fits it.
|
||||
- Done when: one catalog entry with an "ampere+" profile (min 8.0) and a
|
||||
"turing" fallback places the fallback on cuda01, and the same entry,
|
||||
in a test with an A100 in the inventory, places the other.
|
||||
|
||||
### H3. Environments: which models can share a replica (S, the catalog side)
|
||||
- `comfyui.environment: <name>` puts a model in a named environment.
|
||||
Models in one environment must agree on every custom node they both
|
||||
name (same repository, same commit) and their requirements must
|
||||
resolve together; the catalog refuses an environment that cannot, with
|
||||
both models' names. Models with no environment are each their own, as
|
||||
today.
|
||||
- `Catalog.Environments()` returns each environment with its models and
|
||||
the union of their nodes: what one replica of it must install.
|
||||
- Placement and eviction of a replica that serves every model of its
|
||||
environment (so flux, SF3D and Qwen-Image-Edit stop handing the card to
|
||||
each other with a 30 s cold start each time) is the other session's
|
||||
work (llmgw-b7), built on this.
|
||||
|
||||
### H4. Custom nodes with packages that are built, not downloaded (S) — built 2026-10-11
|
||||
Built as written, with `uv build --no-build-isolation` against ComfyUI's
|
||||
Python, the card's capability asked of torch in the replica (so
|
||||
CUDA_VISIBLE_DEVICES is the card it will run on), and wheels kept under
|
||||
the engines' cache by commit, `sm<arch>` and directory. Also
|
||||
`comfyui.huggingface_repo`: a node that loads its weights by repository
|
||||
name (SF3D's) finds them in the engines' Hugging Face cache, linked by the
|
||||
gate, so no token is on the node.
|
||||
- `nodes[].build: [subdir, ...]`: directories of the node's repository
|
||||
that are Python packages with C++/CUDA extensions (SF3D's texture_baker
|
||||
and uv_unwrapper). The gate builds each as a wheel with the engine's
|
||||
Python, `TORCH_CUDA_ARCH_LIST` set from the compute capability of the
|
||||
card it runs on, and installs it into the replica's own `site` like
|
||||
any other requirement. Built wheels are kept under the engine's home by
|
||||
node commit, subdirectory and architecture, so the second replica on
|
||||
the same kind of card does not build again.
|
||||
- A node needing nvcc on a node without it refuses the replica with that
|
||||
said, not with a compiler's traceback.
|
||||
- Done when: SF3D runs on cuda01 with nothing installed into the engine's
|
||||
venv by hand (the hand install of 2026-10-10 removed first).
|
||||
|
||||
### H5. Licence on every output (S) — done 2026-10-14: the whole licence in GLB asset.extras and PNG iTXt; video and audio not yet
|
||||
- `License` gains `id` (SPDX where there is one, else a lowercase name:
|
||||
stability-community, tencent-hunyuan-community), `commercial` (unset is
|
||||
not said), `excluded_regions` (ISO 3166 codes, `EU` for the union),
|
||||
`revenue_cap_usd` and `attribution` ("Powered by Stability AI"). All
|
||||
optional but the name. Done 2026-10-11 in the catalog, so H6 can match
|
||||
on `id`.
|
||||
- Written into the file where the format has a place for it: glTF
|
||||
`asset.extras.license` (as now), PNG an `iTXt` chunk named `license`
|
||||
(the workflow's own text chunks are still stripped), MP4/WebM and the
|
||||
audio formats in their metadata where ffmpeg is at hand on the router,
|
||||
otherwise beside it in the job's answer only — said in the docs.
|
||||
|
||||
### H6. Licence choices (S) — done 2026-10-11 (4bbac08), live on the PoC
|
||||
The user asked (2026-10-11) for the choice to be a person's: on their
|
||||
profile they pick the licences they will not use, and their keys and
|
||||
portal session neither see those models in `/v1/models` nor may call
|
||||
them (403, saying which licence). Matched on `license.id`, and on
|
||||
`commercial` / `excluded_regions` where a choice is put that way. Its
|
||||
store table is migration 0046. A site-wide or team-wide default can sit
|
||||
on top of it later.
|
||||
|
||||
### H7. A job worker per model, not one for all (S) — done 2026-10-14: one job per model at a time, eight in all
|
||||
The worker runs one job at a time: a job waiting for the card holds up a
|
||||
job for another model that could run now (seen on the PoC, 2026-10-10).
|
||||
Jobs are claimed and run concurrently up to a limit per model (or per
|
||||
environment, after H3) and a cluster-wide cap, with the lease and
|
||||
leader-only rules of G1 kept.
|
||||
|
||||
### H8. Presets that add themselves (S) — done 2026-10-13 (v0.14.5)
|
||||
flux-schnell and triposr come with llmgw (`ctl model add <preset>`), and
|
||||
the leader adds them on demand where a node fits one, the cluster installs
|
||||
ComfyUI and the catalog makes nothing of that kind (`presets.auto`); a
|
||||
preset removed is declined and not added back. The cluster page and the
|
||||
console show and add them.
|
||||
|
||||
### H9. Small things learned on the PoC (2026-10-14)
|
||||
- Done: `albedo_match` on sf3d (its delighting made dark objects light);
|
||||
`on_demand.evict_after_idle` / `min_lifetime` as live settings; an
|
||||
upgrade that leaves nodes behind says which; `ctl model serve` serves an
|
||||
embedding model for embedding; refreshes wait while a form is filled in.
|
||||
- Open: llmgw's `/mcp` with picture, 3D, VLM and workflow tools, so a
|
||||
client needs no server of its own (asked by the game team); the shared
|
||||
ComfyUI replica (H3's placement side).
|
||||
|
||||
## D. Facts worth knowing before starting
|
||||
|
||||
- Admin API: `127.0.0.1:9090`, no auth, `ctlClient` in `internal/cli/ctl.go`.
|
||||
|
||||
+645
-1
@@ -17,11 +17,655 @@ is only released once its **exit criterion** is met, not when its code lands.
|
||||
| v0.10.0 | — | InfiniBand fabrics and NVLink, doctor, node add, engines installed by the nodes | released |
|
||||
| v0.11.0 | — | High availability from setup, the setup and console screens, usage sums and per-tenant alerts | released |
|
||||
| v0.12.0 | — | The cluster's settings in the store, in force when changed; the firewall switch; wipe | released |
|
||||
| v1.0.0 | 5 | Tenancy, prefix affinity, benchmarking | planned |
|
||||
| v0.13.0 | — | The whole system stopped and started from one command; Qwen3Guard judges, on a GPU and CPUs together | released |
|
||||
| v0.14.0 | — | Knowledge bases: documents from people's own machines, kept apart per tenant, answered from | released (v0.14.1: adapters, the tree over the API, the firewall on every machine; v0.14.2: pictures from ComfyUI; v0.14.3: 3D models as jobs) |
|
||||
| v1.0.0 | 5 | What phase 5 has left: precise KV routing proven on vLLM, the block cost picker, defragmentation across NVLink domains, prefill/decode disaggregation | planned |
|
||||
|
||||
Until v1.0.0 the minor version is the breaking one: configuration formats and Go
|
||||
APIs change between minors without a deprecation period.
|
||||
|
||||
## Unreleased
|
||||
|
||||
### Added
|
||||
|
||||
- **Pictures uploaded to work from**: `/v1/files` takes `purpose=vision` or
|
||||
`user_data` besides `batch`, kept as long as job results, so a picture is
|
||||
sent once and named by its `file-…` id in the MCP tools and a workflow's
|
||||
inputs.
|
||||
- **One ComfyUI for an environment's models**: ComfyUI models in one
|
||||
`comfyui.environment` (two or more) share one replica — the catalog
|
||||
makes it a hidden entry, `comfyui-env-<name>`, on the first member's
|
||||
profiles with the most memory any member needs, carrying every member's
|
||||
weights and custom nodes. Asking for any member starts it; it answers for
|
||||
each with that member's workflow and checkpoint, and the members get no
|
||||
replicas of their own. A picture model and a 3D model on one card stop
|
||||
handing it to each other with a cold start each way.
|
||||
- **Pictures, 3D models, vision and workflows as MCP tools** at `/mcp`,
|
||||
beside the memory notes, each offered only where the gateway has a model
|
||||
for it: `generate_image` (with `concept`, `side` and `texture` styles),
|
||||
`generate_mesh` (from a picture or a prompt), `review_images` (a vision
|
||||
model, to a JSON schema), `run_workflow`, `job_status` and
|
||||
`gateway_status`. Each makes its request under the caller's key, so
|
||||
allowlists, limits, licences and budgets apply and it is charged like
|
||||
any request; what it makes is kept as the caller's file and answered by
|
||||
id and download URL. A client that takes a stream gets
|
||||
`notifications/progress` while a mesh is made and waits up to ten
|
||||
minutes; one that does not, 50 seconds, then the job id.
|
||||
- **Setup asks where alerts go**: `llmgw server init --webhook URL`
|
||||
(repeatable) and a **Notification webhooks** field on both setup screens.
|
||||
Slack's, Mattermost's and Rocket.Chat's addresses are sent Slack's text,
|
||||
any other the event as JSON (`slack:` or `json:` in front says which).
|
||||
They are added to the cluster's `notify.webhooks` beside those already
|
||||
there, and each is sent a test once the control plane runs on them.
|
||||
|
||||
- **Extended thinking on `/v1/messages`**: `thinking: {type: "enabled"}`
|
||||
tells a model that thinks by its chat template to (`enable_thinking`),
|
||||
and its reasoning comes back as `thinking` blocks before the text,
|
||||
streamed as `thinking_delta` events; `disabled` tells it not to. Not
|
||||
asked for, the reasoning is left out as before.
|
||||
- **How long on-demand models hold a card** is a setting, in force on the
|
||||
leader within seconds: `on_demand.evict_after_idle` (2m unless set) and
|
||||
`on_demand.min_lifetime` (5m). On one card shared by a picture model and
|
||||
a 3D model, 1m each brings the wait between a picture and its mesh to
|
||||
about a minute.
|
||||
|
||||
### Fixed
|
||||
|
||||
- A model whose weights and context pool did not fit its card was added
|
||||
and then failed to make its context on every start, taking the card from
|
||||
others each time (Apertus 70B with a 65536-token pool, 8B with 131072, on
|
||||
24 GB). `ctl model serve` shrinks the pool to what the card holds, saying
|
||||
so, and refuses weights that do not fit at all; the portal's serve form
|
||||
refuses such a pool and says the largest that fits.
|
||||
- Removing weights from the mirror that a catalog model still loads is
|
||||
refused, naming the model: removed from the mirror alone, the model stayed
|
||||
in the catalog and its replicas went on loading the node's copy.
|
||||
- Jobs ran one at a time, so a 3D job waiting for the card held up another
|
||||
model's job that could have run. They run side by side, one per model at
|
||||
a time and eight in all, on the leader as before.
|
||||
- `llmgw ctl model serve` served an embedding model as a chat model: no
|
||||
`--embedding`, four slots, a 512-token batch, `embeddings` unset and a
|
||||
"say hello" smoke test it could not pass. A model whose GGUF says it
|
||||
pools (or `--embeddings`) is served for embedding: 32 slots of 1024
|
||||
tokens unless sized otherwise, a batch that holds a whole passage, and
|
||||
one vector asked for as its smoke test.
|
||||
- `llmgw ctl upgrade` checks every node's version once the waves are done
|
||||
and fails naming any not on the release, and says when the control plane
|
||||
has no nodes registered; `llmgw upgrade` on a control plane says how many
|
||||
nodes it left behind and the command that upgrades them. A site's nodes
|
||||
stayed a release behind with nothing to say so.
|
||||
- The cluster and node pages swapped their live part every few seconds,
|
||||
forms and all, so filling one in raced the clock: what was typed, an
|
||||
open choice and the focus were lost. The refresh now waits while a field
|
||||
in it has the focus or a form in it has been typed into and not sent;
|
||||
the mirror page's reload during a download waits the same way.
|
||||
|
||||
## v0.14.5 — 2026-10-14
|
||||
|
||||
Picture and 3D models that add themselves where a card fits them, with a
|
||||
card on the portal and a screen in the console to see and add them; a
|
||||
mesh's colour brought to the picture it was made from; and the whole
|
||||
licence written into every mesh.
|
||||
|
||||
**Upgrading.**
|
||||
- The store gains a table (migration 0047: preset_state); nothing to do.
|
||||
- **`presets.auto` is on**: once upgraded, the leading control plane adds
|
||||
flux-schnell and triposr, on demand, where a node's card fits one, the
|
||||
cluster installs ComfyUI (`engines.comfyui`) and the catalog makes
|
||||
nothing of that kind. Nothing is fetched until somebody asks for one.
|
||||
Turn it off with `llmgw ctl model presets auto off`.
|
||||
- **Nodes** need this release for `albedo_match` (it is in the save node
|
||||
the gate puts in each replica); a ComfyUI replica takes it when it
|
||||
starts again.
|
||||
|
||||
### Added
|
||||
|
||||
- **A mesh's base colour brought to its picture's**: `LLMGWSaveMesh` takes
|
||||
`albedo_match` with the picture and its mask, and scales the texture in
|
||||
linear light to the object's brightness in the picture. Stable Fast 3D
|
||||
takes the light out of what it sees and read a dark basalt (0.17) as a
|
||||
light stone dimly lit (0.67); matched, it is 0.17. On sf3d it is the
|
||||
`albedo_match` input (off unless asked).
|
||||
- **Picture and 3D models by themselves**: once a node has a card that fits
|
||||
flux-schnell or triposr and the catalog makes nothing of that kind, the
|
||||
leading control plane adds the preset on demand — so a new cluster's
|
||||
portal has its Images and 3D pages without a definition written — where
|
||||
the cluster installs ComfyUI. `presets.auto` turns it off; a preset
|
||||
somebody removes is not added back. `GET /api/v1/presets` says how each
|
||||
stands and why one is not added; `POST /api/v1/presets/{name}` adds one
|
||||
(and sets `engines.comfyui` if the cluster has none); `PUT
|
||||
/api/v1/presets/auto`. Each automatic addition is in the audit log.
|
||||
- **Presets on the portal and in the console**: the cluster page's
|
||||
**Pictures and 3D models** card lists each preset — what it makes, the
|
||||
classes it fits here, whether and by whom it was added, why the leader
|
||||
has not — with an **Add** button that says first when it would install
|
||||
ComfyUI and restart the control planes, and the switch for `presets.auto`.
|
||||
The console's models screen shows the same; **A** adds one, **P** flips
|
||||
the switch. `ctl model add PRESET` and `ctl model presets` go through the
|
||||
API (and `ctl model presets auto on|off`), falling back to the entry as
|
||||
v0.14.4 sent it on an older control plane.
|
||||
|
||||
### Fixed
|
||||
|
||||
- A mesh carried only its licence's name, URL and notice; it carries the
|
||||
whole licence as `/v1/models` gives it — id, commercial use, excluded
|
||||
regions, revenue cap, attribution — for a pipeline that sorts assets by
|
||||
licence.
|
||||
- A workflow's `{{name|false}}` default filled a BOOLEAN input with the
|
||||
word rather than a boolean.
|
||||
|
||||
|
||||
## v0.14.4 — 2026-10-13
|
||||
|
||||
Any ComfyUI workflow as a job, with typed inputs; picture and 3D models
|
||||
added by name; profiles that fit whatever card a site has, and ComfyUI on
|
||||
a V100; licences a machine can match and a person can leave out; a job
|
||||
waiting for the one card says what holds it. Many fixes to on-demand
|
||||
models taking turns on one card.
|
||||
|
||||
**Upgrading.**
|
||||
- The store gains a table (migration 0046: user_excluded_licences);
|
||||
nothing to do.
|
||||
- **Nodes** need this release for ComfyUI on Volta or Pascal cards (the
|
||||
cu126 build), for a driver from NVIDIA's open kernel module to be read,
|
||||
and for a node that loads its weights by repository name (sf3d); a
|
||||
ComfyUI already installed is kept.
|
||||
- **Control planes before nodes**: an entry using the built-in `triposr`
|
||||
workflow, or a `gpu:` need instead of a class, is refused by an older
|
||||
control plane.
|
||||
- Models added with `llmgw ctl model add <preset>` take the preset's name;
|
||||
remove an entry of that name first.
|
||||
|
||||
### Added
|
||||
|
||||
- **The client API described**: OpenAPI 3.1 at `/v1/openapi.json`, no key
|
||||
needed, generated from a route table a test holds to the router's own;
|
||||
and `/docs/api`, Swagger UI (served from the gateway) over it and, for
|
||||
staff, over the admin API's document.
|
||||
- **A job waiting for the card says so**: `progress` is
|
||||
`{"state":"waiting_gpu","held_by":…,"eta_seconds":…}` while another
|
||||
on-demand model holds it; a direct request's 503 carries
|
||||
`X-Llmgw-Waiting-For` and `X-Llmgw-Eta-Seconds`.
|
||||
- **Any ComfyUI workflow, as a job**: `POST /v1/workflows/runs` with
|
||||
`{model, inputs}`. A model's catalog entry declares its inputs
|
||||
(`comfyui.inputs`: text, int, float, bool, choice, image, mask, audio,
|
||||
video, file, with bounds and defaults) and its output (`comfyui.output`:
|
||||
pictures, meshes, video, audio or files); a request is checked against
|
||||
them before a job is written, file inputs are base64, a data URL or a
|
||||
`/v1/files` id, and `/v1/models` lists each model's `workflow` so a
|
||||
client can build its form. A PNG result carries the licence in an
|
||||
`iTXt` chunk, a GLB in `asset.extras` as before.
|
||||
- **Environments**: `comfyui.environment` names ComfyUI models that agree
|
||||
on their custom nodes and may share one replica; the catalog refuses
|
||||
one that cannot.
|
||||
- **Profiles that fit the hardware**: a profile may say `gpu:
|
||||
{min_compute_capability, min_memory_gb}` instead of `gpu_class`, and
|
||||
stands for every class in the cluster that meets it, as
|
||||
`<profile>@<class>`; a ComfyUI profile may carry its own `comfyui`
|
||||
workflow, checkpoint and nodes, and any profile its own `source`. One
|
||||
catalog entry then runs fp8 where the card has it and fp16 on Turing.
|
||||
- **Custom nodes that compile**: `nodes[].build` names directories of a
|
||||
node that are C++/CUDA packages; the gate builds each as a wheel for the
|
||||
card the replica runs on, keeps it on the node, and installs it beside
|
||||
the replica. `comfyui.huggingface_repo` shows a model's weights to a node
|
||||
that loads them by repository name.
|
||||
- **A licence a machine can match**: `license.id`, `commercial`,
|
||||
`excluded_regions`, `revenue_cap_usd`, `attribution`.
|
||||
- **On-demand models in `/v1/models`**, with `loaded` saying whether one is
|
||||
up now; a picture is cut out and put on grey before TripoSR sees it
|
||||
(`LLMGWPrepareForMesh`, rembg when it has no alpha).
|
||||
- **Licences a person leaves out**: on the profile page, by licence, or by
|
||||
rule — every licence forbidding commercial use, every licence granting
|
||||
nothing in a region — so a model under one is neither listed to nor
|
||||
served for that person's keys and sessions (403 `licence_excluded`).
|
||||
- **Presets: a picture or 3D model added by name.** `llmgw ctl model add
|
||||
flux-schnell` or `triposr` adds an entry that comes with llmgw — its
|
||||
workflow (TripoSR's now built in), custom nodes, packages and licence,
|
||||
and a profile by card need rather than class — and has the nodes install
|
||||
ComfyUI when the cluster's settings name none. `llmgw ctl model presets`
|
||||
lists them; the console's **A** on the models screen offers them beside
|
||||
a definition file.
|
||||
|
||||
### Fixed
|
||||
|
||||
- A model waiting for another on-demand model to give up the card failed at
|
||||
once as "cannot be placed"; it now waits, and a job with it.
|
||||
- A model evicted for being idle was placed again on the next pass, ahead of
|
||||
the one it made room for.
|
||||
- A job or batch leased by a standby control plane, whose router has no
|
||||
endpoints, waited on "loading" for good; they are served only by the
|
||||
leader. A job told its model fits nowhere asks again for five minutes
|
||||
before failing, which an agent re-registering makes true for seconds.
|
||||
- An on-demand replica adopted after a restart was never idle enough to
|
||||
evict, so a model waiting for its card waited out its whole idle timeout;
|
||||
it counts as used at adoption.
|
||||
- A picture in a prompt was counted as its base64 by the context check, so
|
||||
one 1280x720 PNG was "138 thousand tokens" and a vision model refused it;
|
||||
a picture or a sound now counts as about a thousand.
|
||||
- An on-demand model of a higher class was evicted, idle, for a lower-class
|
||||
one two minutes into a lull; it keeps the card through its idle timeout,
|
||||
and a model waiting on it is told so.
|
||||
- A model removed from the catalog stayed wanted, so added again it loaded
|
||||
unasked.
|
||||
- A node that loads its weights by repository name (sf3d) went to Hugging
|
||||
Face for a gated repository and got a 401: the agent did not hand the
|
||||
gate `comfyui.huggingface_repo`, so the weights on the node were never
|
||||
linked into its cache.
|
||||
- **ComfyUI on a V100.** torch's CUDA 12.8 and newer builds have no Volta
|
||||
kernels, but its cu129 build was taken to run from compute capability
|
||||
7.0, so on a V100 both ComfyUI builds were installed, about 10 GB each,
|
||||
and failed with "no kernel image". Cards older than Turing now get
|
||||
ComfyUI on PyTorch's CUDA 12.6 torch (build `cu126`); a Turing or newer
|
||||
node keeps the build it has.
|
||||
- **The open NVIDIA driver read by its version.** With the open kernel
|
||||
module, the default since 560, the agent read the driver as "for" and so
|
||||
as unknown, and tried every engine build whatever driver it needs.
|
||||
|
||||
## v0.14.3 — 2026-10-10
|
||||
|
||||
3D models from a picture, as jobs: made by ComfyUI on the cluster's cards,
|
||||
with their progress, their licence written into the mesh, a price per mesh
|
||||
and a page to make them. Also: a profile's own weights, a band for bulk
|
||||
work, and fixes for control planes behind a balancer and agents that
|
||||
restart.
|
||||
|
||||
**Upgrading.**
|
||||
- The store gains tables and columns (migrations 0043–0045: notify_sent,
|
||||
jobs, usage meshes); nothing to do.
|
||||
- **Nodes** need this release for their engines' output to survive an
|
||||
agent restart and for ComfyUI's custom nodes and progress; engines
|
||||
started by an older agent keep its pipe until they are restarted.
|
||||
- A 3D model needs a model that makes them (see install.md, "A model that
|
||||
makes them"); none is added.
|
||||
|
||||
### Added
|
||||
|
||||
- **Jobs: 3D models asked after, not waited for.** `POST
|
||||
/v1/3d/generations` answers 202 with a job id; `GET
|
||||
/v1/3d/generations/{id}` says its status, progress and, when done, the
|
||||
file its result is in (`/v1/files/{id}/content`); `POST .../cancel` stops
|
||||
it, in the engine's queue too. The request is replayed by whichever
|
||||
gateway holds the job's lease through the same path a client's request
|
||||
takes — key, allowlist, limits, admission, guardrails, accounting — so
|
||||
it is charged like any request, tagged `job=<id>`, and a model still
|
||||
loading is waited out for as long as the job has (24 h) rather than
|
||||
refused. A job survives the gateway that took it; results are kept for
|
||||
`router.batch.retain_days`. Migration 0044.
|
||||
- **ComfyUI models that make meshes** (`comfyui.output: meshes`). The
|
||||
picture is uploaded to the replica's ComfyUI and the workflow's
|
||||
`{{image}}`; the mesh the workflow saves is the job's file, an OBJ turned
|
||||
into a GLB. TripoSR (MIT) is the tested model; on Ampere and newer cards
|
||||
stronger ones can be added the same way.
|
||||
- **Custom nodes** (`comfyui.nodes`): a repository at a full commit, put in
|
||||
each replica's own `custom_nodes` by the gate, with its packages
|
||||
installed beside the replica, never into ComfyUI's environment.
|
||||
- **A job's progress**: the gate listens to ComfyUI's websocket and the job
|
||||
says queued and how many ahead, then the node running, its step and
|
||||
percent.
|
||||
- **A model's `license`** (name, url, notice), in `/v1/models`, in the job,
|
||||
on the portal and in the GLB's `asset.extras`.
|
||||
- **`per_mesh`** in `auth.pricing`; usage rows, sums and exports count
|
||||
`meshes`.
|
||||
- **`/3d` on the portal**, beside `/images`: a picture chosen, the job
|
||||
followed, the mesh in a viewer with a download and its licence; and
|
||||
**`llmgw ctl 3d list|get|cancel`** for every owner's jobs.
|
||||
- **A profile's own `source`**: the weights it loads instead of the
|
||||
model's, so an embedding model can run on vLLM from its repository on a
|
||||
card and on llama.cpp from a GGUF on CPUs as one model. Placement prefers
|
||||
nodes holding each profile's weights, `ctl model add` checks every
|
||||
source, and `ctl weights` collects none a profile loads.
|
||||
- **A band's own `queue_ttl`** (`router.admission.bands[].queue_ttl`): how
|
||||
long its requests may wait for room, instead of the admission's 30 s. A
|
||||
band for bulk work — a knowledge-base import — waits behind people rather
|
||||
than being turned away.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **An engine's output no longer breaks when its agent restarts.** Engines
|
||||
wrote to a pipe of the agent's, which an agent that restarted and adopted
|
||||
them left with no reader; an engine's next large write failed, and in a
|
||||
Python engine took the request it was reporting on with it. Each process
|
||||
now writes to its own file in the agent's work directory, and an adopted
|
||||
engine's log keeps what it wrote before the restart.
|
||||
- **Single sign-on behind a balancer.** The cookie that carries a sign-in
|
||||
across the identity provider was signed with a key of each process's own,
|
||||
so with two control planes a callback that reached the other one was
|
||||
refused. The key is now a secret in the store, the same on every control
|
||||
plane.
|
||||
- **Notifications sent once by two control planes.** Each kept its own
|
||||
hour of quiet per event, so a budget spent on a team both served was
|
||||
announced twice. The control plane that claims an event in the store
|
||||
(migration 0043) sends it; with the store unreachable it is sent anyway.
|
||||
- **A request turned away for a busy cluster says when to come back.** A
|
||||
request that waited as long as it may (504) or found the queue full (503)
|
||||
had no `Retry-After`, so `llmgw-kb` counted the passage as failed instead
|
||||
of sending it again; both now say 10 seconds.
|
||||
- The usage sums and exports count pictures (`images`), as the request
|
||||
records did; pictures made before are added to the sums.
|
||||
|
||||
## v0.14.2 — 2026-10-10
|
||||
|
||||
Pictures: OpenAI's `/v1/images/generations`, made by ComfyUI on the
|
||||
cluster's own cards. The agents install ComfyUI themselves, the gateway
|
||||
prices pictures by the picture, and the portal has a page to make them.
|
||||
Also: on-demand models that survive a failover, and knowledge-base
|
||||
embeddings fixes.
|
||||
|
||||
**Upgrading.**
|
||||
- The store gains a column (`usage.images`); nothing to do.
|
||||
- **ComfyUI is installed only where asked**: `llmgw ctl config set
|
||||
engines.comfyui 0.39.2` has every node with an NVIDIA GPU build it (about
|
||||
10 GB under `/opt/llmgw/engines`). Nodes need this release to install it
|
||||
or run engine `comfyui`.
|
||||
|
||||
### Added
|
||||
|
||||
- **Pictures at `/v1/images/generations`**, OpenAI's shape, through the same
|
||||
key, allowlist, limit, admission and guardrail checks as any request. A
|
||||
model with profiles on engine `comfyui` is placed on a GPU node like any
|
||||
other. A ComfyUI server outside the cluster can be an external model with
|
||||
`api: comfyui` instead. The workflow is a built-in (`flux-schnell`, the
|
||||
default, or `flux-dev`) or a JSON file in ComfyUI's API format, with
|
||||
`{{prompt}}`, `{{width}}`, `{{seed}}` and the like where the request's
|
||||
values go. Requests may also send `seed`, `steps`, `negative_prompt` and
|
||||
`guidance`. The seed used comes back in `x-llmgw-seed`. Pictures come back
|
||||
as `b64_json`; ComfyUI's workflow is stripped from each PNG, and a client
|
||||
that leaves takes its workflow off the queue. `/v1/models` says `images`
|
||||
for such a model.
|
||||
- **Engine `comfyui`.** ComfyUI asks for no key, so the agent starts it
|
||||
behind `llmgw comfyui-gate`. ComfyUI listens on the loopback only. The
|
||||
gate listens on the replica's port, demands the replica's key, and lets
|
||||
through only what the router asks. The model's `comfyui:` block names
|
||||
the workflow and, for a source that is a set of files, the checkpoint.
|
||||
- **The agents install ComfyUI** (`engines.comfyui`). The release's source is
|
||||
checked against a digest llmgw holds, since GitHub publishes none, and a
|
||||
version without one is refused. Its requirements are resolved as PyPI
|
||||
stood the day after the release, with PyPI's CUDA 13 torch or PyTorch's
|
||||
CUDA 12.9 one for an older driver, and the result is checked on the card.
|
||||
A ComfyUI installed by hand is named by `engines.comfyui.dir` in
|
||||
agent.yaml.
|
||||
- **A price per picture**: `per_image` in `auth.pricing`, on top of the
|
||||
prompt's tokens. A model priced only per picture is not free, so a spent
|
||||
budget still stops it. Each usage row keeps how many pictures were made,
|
||||
shown by `llmgw ctl request`.
|
||||
- **`/images` on the portal**, beside the chat, for the models that make
|
||||
pictures. The chat no longer offers them. The page offers size, how many,
|
||||
seed, steps and a negative prompt, and shows each picture with its seed
|
||||
and a download link. Nothing is kept but the usage.
|
||||
- **An embedding model's `query_prefix`**, put before the question a
|
||||
knowledge-base search embeds and never before documents, for Qwen3-Embedding
|
||||
and models trained like it.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **An awake on-demand model survived no restart or change of leader of the
|
||||
control plane.** A control plane that had heard no requests read every
|
||||
on-demand replica its nodes reported as wanted by nobody, and drained them
|
||||
all. An adopted replica now gets a whole idle timeout.
|
||||
- `ctl models` said `not placed` for an on-demand model that had been woken
|
||||
and was serving; it now says `ready (on demand)`.
|
||||
- A batch of embeddings was refused as one over-long prompt when the
|
||||
passages together exceeded the context, though each one fit. The longest
|
||||
input is what has to fit.
|
||||
- The chat's kept conversation sat in the browser under one key for
|
||||
everybody, so the next person to sign in on the same browser opened the
|
||||
last one's conversation. It is kept per account, and signing out clears
|
||||
what the pages stored.
|
||||
|
||||
## v0.14.1 — 2026-10-09
|
||||
|
||||
Adapters trained on the nodes from a knowledge base and kept private to
|
||||
their owners; the cluster's and the agents' settings edited in one place;
|
||||
the tenancy tree and keys over the admin API and in Terraform; knowledge
|
||||
bases that hold up with many documents; the firewall setting on every
|
||||
machine.
|
||||
|
||||
**Upgrading.**
|
||||
- **Nodes** need this release to follow `agents.*` in the cluster's
|
||||
settings; until a node is upgraded its agent.yaml is what it runs on.
|
||||
- **A separate mirror machine and store members that run nothing else**
|
||||
follow the firewall setting once what reads it is installed: run `sudo
|
||||
llmgw mirror init` on the mirror again with the flags it was set up with,
|
||||
and `llmgw server init --patroni` again for the store's members. A control
|
||||
plane set up before v0.12.0 needs `sudo llmgw firewall sync --install`.
|
||||
- **The store's PostgreSQL memory** applies after each member is restarted,
|
||||
one at a time, the leader last; until then it runs as before.
|
||||
- The store gains tables and columns; nothing to do. Existing knowledge-base
|
||||
collections get their full-text index when a control plane starts, and
|
||||
keep `vector` storage; only new collections use `halfvec`, and only on
|
||||
pgvector 0.7 or later.
|
||||
- Cached knowledge-base passages in vLLM are made again once, under the new
|
||||
`cache_salt`.
|
||||
|
||||
### Added
|
||||
|
||||
- **The cluster's settings on the portal** (BACKLOG E4): `/admin/settings`
|
||||
for administrators. It edits the document `llmgw ctl config edit` opens:
|
||||
checked as a control plane would load it, refused when made from a
|
||||
version since replaced, with the history to view and roll back. Changes
|
||||
are audited by size, never by contents.
|
||||
- **The nodes' agent settings in the cluster's settings** (BACKLOG E3):
|
||||
`agents.default` and `agents.nodes.<id>` hold everything agent.yaml can
|
||||
say but the agent's id, control planes, address and certificate. They are
|
||||
sent to the agents as they change; an agent starts again on new settings,
|
||||
or on an edited agent.yaml, with its engines serving throughout. Settings
|
||||
that would not load are refused on save, and set aside on a node rather
|
||||
than keep its agent from starting. Nodes need this release to follow
|
||||
them.
|
||||
- **LoRA adapters trained on the nodes** (BACKLOG F4): `llmgw ctl kb dataset`
|
||||
writes question/answer pairs from a collection with a model of the
|
||||
cluster. `llmgw ctl train start` trains an adapter on a node with its
|
||||
installed vLLM's torch and a small overlay, on a card with enough memory
|
||||
free (beside an engine, if it leaves that much), converting to GGUF for
|
||||
llama.cpp. `llmgw ctl train publish` keeps it on the mirror (which now
|
||||
accepts uploads) and makes it the model's adapter.
|
||||
- **Adapters with owners.** An adapter's `owner` (org, team, project, user
|
||||
or key) is the only part of the tree that sees it; to anyone else it is a
|
||||
model that does not exist. An adapter trained on a knowledge base belongs
|
||||
to a part of the tree that may read it, and is never made public.
|
||||
- The knowledge bases' vLLM `cache_salt` follows the collections searched,
|
||||
not the organization, so two teams of one organization no longer share
|
||||
cached passages.
|
||||
- **Organizations, teams, projects, API keys and memberships in the admin
|
||||
API**: `/api/v1/orgs` (the tree), `POST /api/v1/teams` and `projects`,
|
||||
`PATCH` to rename or move, `DELETE` for what is empty; `/api/v1/keys` to
|
||||
issue (the key is in the answer, once), list and revoke; `/api/v1/users`
|
||||
and `/api/v1/users/{id}/memberships`. They were only `llmgw org`, `keys`
|
||||
and `users` on a control plane before. The Terraform provider manages them
|
||||
as `llmgw_org`, `llmgw_team`, `llmgw_project`, `llmgw_key` (the key in a
|
||||
sensitive `secret`) and `llmgw_membership`.
|
||||
- **The firewall setting reaches every machine.** A separate mirror and the
|
||||
store's members follow `firewall.enabled` and `firewall_networks` as nodes
|
||||
and control planes did: the leading control plane tells the mirror over
|
||||
its API, and keeps the state in the store for members that run no llmgw,
|
||||
which a timer on each reads. The mirror also lets in the network of every
|
||||
node the leader knows. `mirror init` and `server init --patroni` install
|
||||
what follows it; run them again on a cluster set up before.
|
||||
|
||||
### Changed
|
||||
|
||||
- **The store's PostgreSQL is sized to its machine's memory.** Before, it ran
|
||||
on PostgreSQL's defaults (128MB cache) under Patroni, or on 96MB on one
|
||||
machine. Now each member gets a share of its own RAM, a smaller one where
|
||||
a control plane or agent shares the machine, and the share is read again
|
||||
every time `server init --patroni` runs. The new cache is applied by
|
||||
restarting one member at a time, the leader last.
|
||||
- **Knowledge-base searches hold up with many documents.** New collections
|
||||
store their index as `halfvec` where pgvector is 0.7 or later (half the
|
||||
memory, and embeddings up to 4000 dimensions instead of 2000); older
|
||||
collections, and clusters on pgvector 0.6, keep `vector`. Each collection
|
||||
has its own full-text index, as it has its own vector index, made for
|
||||
existing collections when the control plane starts. The text arm ranks by
|
||||
the cheap measure first and scores only those candidates.
|
||||
|
||||
### Fixed
|
||||
|
||||
- A knowledge-base search asking for more than 10 passages
|
||||
(`X-Llmgw-Knowledge-K`) got at most 40 candidates per collection from the
|
||||
vector index, whatever it asked for: pgvector's `hnsw.ef_search` is now
|
||||
raised to the number wanted, up to 1000.
|
||||
- `llmgw server backup` and `restore` against a replicated store over TLS
|
||||
failed looking for `~/.postgresql/root.crt`: pg_dump was given
|
||||
`sslmode=verify-full` without the cluster's authority.
|
||||
|
||||
## v0.14.0 — 2026-10-08
|
||||
|
||||
Knowledge bases: a company's documents, put in by people from their own
|
||||
machines, kept apart per tenant, and answered from.
|
||||
|
||||
**Upgrading.**
|
||||
- **pgvector on a cluster set up before this release.** Knowledge bases
|
||||
need pgvector, which `server init` and the high-availability database
|
||||
setup now install, but a rerun on a running cluster does not:
|
||||
1. Install `postgresql-<version>-pgvector` on every database machine.
|
||||
2. As postgres, run `CREATE EXTENSION vector` in llmgw's database.
|
||||
3. Restart the control planes.
|
||||
Until then everything else works, and `llmgw ctl kb list` says why
|
||||
knowledge bases are off. On a SQLite store they stay off.
|
||||
- **An embedding model.** A collection needs a catalog model with
|
||||
`capabilities.embeddings`, for example bge-m3, whose weights are in the
|
||||
mirror.
|
||||
- **The client.** People download `llmgw-kb` for their system from this
|
||||
release.
|
||||
|
||||
### Added
|
||||
|
||||
- **Collections and who may read them.** A collection is owned by an
|
||||
organization, team, project or person. Grants add up down the tenancy
|
||||
tree and never cross an organization. A collection a key may not read is
|
||||
answered as one that does not exist. Chunks are kept in PostgreSQL with
|
||||
pgvector, partitioned by organization, with an HNSW index per collection,
|
||||
and row-level security on the table as a second line behind every
|
||||
query's own filter. A project, team or organization that owns collections
|
||||
is not deleted; a person's collections go with their account.
|
||||
- **`llmgw-kb`**, its own download for Windows, macOS and Linux, run by
|
||||
people with their own key. It reads documents where they are and sends
|
||||
text and vectors, never the files, for what changed only; a SQLite ledger
|
||||
on the machine remembers what it did.
|
||||
- It reads PDF, Word, Excel, PowerPoint, OpenDocument, RTF, EPUB, HTML,
|
||||
MHTML, mail with attachments, notebooks, subtitles, ZIP archives, and
|
||||
text and source code in any common encoding.
|
||||
- Legacy Office files are read with LibreOffice, and scanned pages with
|
||||
Tesseract, when those are installed.
|
||||
- `--dry-run` says what would leave the machine; `--delete` removes only
|
||||
what that machine sent.
|
||||
- **Answers from documents.** A chat request names collections in
|
||||
`X-Llmgw-Knowledge`, or its part of the tree has a default (`llmgw ctl kb
|
||||
default`). The best passages, chosen by meaning plus a little for the
|
||||
question's words, are added to the system message, numbered, and
|
||||
`x-llmgw-sources` says which were used. This happens before the
|
||||
guardrails judge the request, and the answer is never served from the
|
||||
answer cache. On vLLM replicas the organization's `cache_salt` keeps
|
||||
tenants' prefix caches apart. It works through the Responses and Messages
|
||||
APIs too.
|
||||
- **Administration**: `llmgw ctl kb` (create, list, grant, revoke, default,
|
||||
docs, rm, search, delete), `/api/v1/kb`, the console's Knowledge screen,
|
||||
and a Documents picker on the portal's chat page.
|
||||
- Metrics `llmgw_kb_retrievals_total` and `llmgw_kb_retrieval_seconds`;
|
||||
[docs/knowledge.md](docs/knowledge.md).
|
||||
|
||||
## v0.13.0 — 2026-10-08
|
||||
|
||||
The whole system stopped, started and restarted from one command, a
|
||||
guardrails judge that no longer refuses ordinary coding requests, and the
|
||||
rest of the field report from v0.10.6.
|
||||
|
||||
**Upgrading.** A cluster keeps the judge its policies name: one installed
|
||||
before this release goes on asking Llama Guard 3 1B until `llmgw ctl guard
|
||||
judge use qwen3guard-0.6b`, which the leader then fetches and serves by
|
||||
itself, and which replaces the policies' Llama Guard category codes.
|
||||
|
||||
### Added
|
||||
|
||||
- **A judge on a GPU and one on CPUs together**: a policy's `judge.models`
|
||||
is a list asked in turn, the first that answers giving the verdict; the
|
||||
next is asked when one has no replica serving, fails, or misses its
|
||||
`latency_budget`. `llmgw ctl guard judge use qwen3guard-4b
|
||||
qwen3guard-0.6b` sets the order for every policy with a judge, and
|
||||
`guard judge status` says which judge stands in for which. A judge on a
|
||||
GPU is served as a `standard` model, so it takes a card only from what
|
||||
may give one up.
|
||||
- **A judge a policy names is fetched if it is missing**: the leader
|
||||
fetches the weights of a judge llmgw knows (`qwen3guard-0.6b`, `-4b`,
|
||||
`-8b`, `llama-guard-3-1b`, `-8b`) into the mirror and serves it, when a
|
||||
policy names it and the catalog does not have it — and again if it is
|
||||
removed. `guard judge enable --gpu` serves Qwen3Guard-Gen 4B on the class
|
||||
with the most free cards.
|
||||
- **The whole system stops, starts and restarts from one command**: `llmgw
|
||||
stop` takes every model down and answers every request for one with 503
|
||||
(`system_stopped`), keeping the control planes, agents and mirror running
|
||||
and no model's definition changed; `llmgw start` brings every model back.
|
||||
It is the cluster's setting `serving.stopped`, live on every control plane.
|
||||
`llmgw restart` restarts every node's agent and every control plane, the
|
||||
leader last, with the engines serving throughout. `llmgw status` says
|
||||
whether the system serves. One machine's services are `llmgw service
|
||||
status|start|stop|restart`.
|
||||
- **A replica says what it computes on**: `ctl replicas` (ON), the console
|
||||
and the admin API show `CUDA` with the GPUs its processes hold memory on
|
||||
and how much, as nvidia-smi sees them, or `CPU` — for every engine, kept by
|
||||
the agent across its own and the control plane's restarts. llama.cpp's
|
||||
layer count is added when it says it, which it does only at `-lv 4`.
|
||||
- **`ctl engine-check llamacpp` computes a token on the GPU**: a small GGUF
|
||||
(`--model`, or the smallest in the weights cache) is loaded with every layer
|
||||
on the GPU at `-lv 4` and asked for one token; a build whose CUDA backend
|
||||
does not load fails, by llama.cpp's own account of its layers. With no
|
||||
`--bin` it checks the build the agent installed.
|
||||
- **Engine versions and compute capability on nodes**: `ctl nodes` shows
|
||||
`llamacpp b10991` and `6000-24g (7.5)`; the admin API gives
|
||||
`engine_versions` and `compute_capabilities`. A profile's `engine_version`
|
||||
restricts it to nodes whose engine version starts with it, and
|
||||
explain-placement says which version a node has, or which engines.
|
||||
- **`ctl restart --max-surge N` and `--max-unavailable N`**: that many of a
|
||||
model's replicas restart at once, beside themselves or stopped first.
|
||||
- **Multi-node readiness apart from the node's**: doctor gives both, with
|
||||
fabric findings in a section of their own; `node add` says the multi-node
|
||||
one; `ctl nodes` shows a declared fabric's adapters that are down.
|
||||
- **A forgotten password is set through a link**: `/forgot` mails one when
|
||||
the cluster's `mail` settings say how (SMTP, sender, the public
|
||||
`base_url` it points at), saying the same whether or not the address has an
|
||||
account; `/reset` sets the password once and closes every session. Without
|
||||
mail, `llmgw users reset-link` makes the link for an operator to hand over.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **A replica restarted in place keeps its room**: the replica was stopped
|
||||
and the pass went on to other models before placing its replacement, so a
|
||||
model waiting for room took the freed cards. On the reference cluster a
|
||||
GPU judge waiting for the only card took it from the model being
|
||||
restarted there. The replacement is now placed in the same step.
|
||||
- **The judge no longer refuses ordinary coding requests**: Llama Guard 3
|
||||
1B, the judge `server init` installed, could not tell "continue", "devam
|
||||
et" or "how do I kill all child processes in bash" (0.51 to 0.92 unsafe)
|
||||
from harmful requests (0.97 to 1.00), at any quantisation. The judge is
|
||||
now Qwen3Guard-Gen 0.6B (format `qwen3guard`), which gave the same
|
||||
ordinary requests 0.15 at the most and the harmful ones 1.00, on CPUs,
|
||||
in 0.8 GB. Its third verdict, controversial, counts as safe unless the
|
||||
policy says `controversial: unsafe`. `guard judge use qwen3guard-0.6b`
|
||||
moves an installed cluster to it, and replaces Llama Guard's category
|
||||
codes, which Qwen3Guard never names, with the baseline's.
|
||||
- **A long message is judged rather than refused**: a judge read the whole
|
||||
last user message, and one with a file pasted into it overflowed the
|
||||
judge's context, so it gave no verdict and `fail: closed` refused the
|
||||
request. It now reads the start and the end, about 6000 characters.
|
||||
- **A replica no longer stays "starting" while it serves after a restart**:
|
||||
an agent that registered before its adopted engines had answered their
|
||||
health check never said they had, so the control plane kept them starting
|
||||
and routed nothing to them — on the reference cluster, after `llmgw
|
||||
restart`, the only replica of a model. The agent now reports each shard
|
||||
the registration called not ready once it is.
|
||||
- **A failover no longer replaces healthy replicas**: a standby that took
|
||||
over placed at once, before the agents had reconnected and said what runs,
|
||||
so it started a second copy of replicas it did not know of and then drained
|
||||
the healthy ones as surplus — on the reference cluster the guardrails'
|
||||
judge and a model's replica, within twenty seconds of a leader restarting.
|
||||
Placement now waits after a takeover as it does at start.
|
||||
|
||||
### Changed
|
||||
|
||||
- **A GPU placement llama.cpp says it cannot compute on is refused** when it
|
||||
warns that it found no usable GPU, even where nvidia-smi cannot be asked;
|
||||
where it can, holding no memory on its GPUs refuses it as before.
|
||||
|
||||
## v0.12.2 — 2026-10-06
|
||||
|
||||
The guardrails turned off and on as a whole, and a judge that refuses
|
||||
|
||||
@@ -6,6 +6,7 @@ LDFLAGS := -X '$(MODULE)/internal/version.Version=$(VERSION)'
|
||||
|
||||
build:
|
||||
go build -ldflags "$(LDFLAGS)" -o bin/llmgw ./cmd/llmgw
|
||||
go build -ldflags "$(LDFLAGS)" -o bin/llmgw-kb ./cmd/llmgw-kb
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
@@ -3,8 +3,9 @@
|
||||
A single-binary LLM gateway for clusters with multiple InfiniBand fabrics,
|
||||
NVLink domains and heterogeneous GPUs.
|
||||
|
||||
It serves LLMs running on your own GPU servers (vLLM / SGLang) behind one
|
||||
OpenAI-compatible endpoint. Design document: [llmgw-plan.md](llmgw-plan.md).
|
||||
It serves LLMs running on your own GPU and CPU servers (vLLM, SGLang,
|
||||
llama.cpp, TensorRT-LLM) behind one OpenAI- and Anthropic-compatible
|
||||
endpoint, and answers from your own documents. Design document: [llmgw-plan.md](llmgw-plan.md).
|
||||
|
||||
## Getting started
|
||||
|
||||
@@ -29,7 +30,7 @@ Releases track the roadmap phases; see [CHANGELOG.md](CHANGELOG.md).
|
||||
| v0.1.0 | 0 | Skeleton and sim-engine | released |
|
||||
| v0.2.0 | 1 | Static router (filter/score/pick, streaming proxy) | released |
|
||||
| v0.3.0 | 1.5 | Admission control (priority bands) | released |
|
||||
| v0.4.0 | 2 | Agent: GPU/IB discovery, engine supervision | released |
|
||||
| v0.4.0 | 2 | Agent: GPU/IB discovery, engine supervision | released as part of v0.5.0 |
|
||||
| v0.5.0 | 3 | Control plane: catalog, placement, reconciler, ctl | released |
|
||||
| v0.6.0 | 4 | Multi-node placement and fabric awareness | released |
|
||||
| v0.7.0 | — | Conversations, agent memory, disk management | released |
|
||||
@@ -37,45 +38,106 @@ Releases track the roadmap phases; see [CHANGELOG.md](CHANGELOG.md).
|
||||
| v0.9.0 | — | Managing people and the tenancy tree | released |
|
||||
| v0.10.0 | — | InfiniBand fabrics and NVLink, doctor, node add, engines installed by the nodes | released |
|
||||
| v0.11.0 | — | High availability from setup, the setup and console screens, usage sums and per-tenant alerts | released |
|
||||
| v1.0.0 | 5 | Tenancy, prefix affinity, benchmarking | planned |
|
||||
| v0.12.0 | — | The cluster's settings in the store, in force when changed; the firewall switch; wipe | released |
|
||||
| v0.13.0 | — | The whole system stopped and started from one command; Qwen3Guard judges, on a GPU and CPUs together | released |
|
||||
| v0.14.0 | — | Knowledge bases: documents from people's own machines, kept apart per tenant, answered from | released (v0.14.1: adapters, the tree over the API, the firewall on every machine; v0.14.2: pictures from ComfyUI; v0.14.3: 3D models as jobs; v0.14.4: any ComfyUI workflow as a job, presets, licences; v0.14.5: picture and 3D models that add themselves) |
|
||||
| v1.0.0 | 5 | What phase 5 has left: precise KV routing proven on vLLM, the block cost picker, defragmentation across NVLink domains, prefill/decode disaggregation | planned |
|
||||
|
||||
As of v0.6.0 a cluster runs unattended: agents report their hardware, the
|
||||
control plane places replicas — including ones larger than a single machine —
|
||||
models load when someone asks for them and stop when nobody does, and keys,
|
||||
budgets and rate limits decide who may use any of it.
|
||||
|
||||
Since v0.11.0, not yet released (see [CHANGELOG.md](CHANGELOG.md),
|
||||
"Unreleased"):
|
||||
Since v0.11.0 (each release's detail is in [CHANGELOG.md](CHANGELOG.md)):
|
||||
|
||||
- **Security hardening** — signed releases only, a token on the admin API,
|
||||
engines not run as root, TLS for the agent port, etcd and Patroni, a
|
||||
firewall for llmgw's internal ports, encrypted backups, bounded
|
||||
connections, sign-in throttling, a Content-Security-Policy.
|
||||
- **Guardrails** — policies that check requests and answers (PII, secrets,
|
||||
prompt injection, deny patterns, a judge model such as Llama Guard), in
|
||||
observe or enforce mode, attached to any part of the tenancy tree:
|
||||
- **Picture and 3D models that add themselves** (v0.14.5). Where a node's
|
||||
card fits flux-schnell or triposr and the catalog makes nothing of that
|
||||
kind, the leader adds it on demand, so a new cluster's portal has its
|
||||
Images and 3D pages; the cluster page and the console show each preset
|
||||
and add it by hand, and `presets.auto` turns it off. Stable Fast 3D's
|
||||
colours can be brought to the picture's (`albedo_match`), and every mesh
|
||||
carries its whole licence.
|
||||
- **Any ComfyUI workflow, and models by name** (v0.14.4). `POST
|
||||
/v1/workflows/runs` runs a ComfyUI model's workflow as a job, its inputs
|
||||
(text, numbers, choices, images, masks, audio, video, files) declared in
|
||||
the catalog and listed in `/v1/models`. `llmgw ctl model add triposr` or
|
||||
`flux-schnell` adds a tested model with no YAML; a profile may name the
|
||||
card it needs instead of a class, and ComfyUI now installs on V100s. A
|
||||
licence carries what a machine can match, and a person can leave
|
||||
licences out on the profile page so no key of theirs reaches such a
|
||||
model. A 3D job waiting for the one card says what holds it and for
|
||||
about how long.
|
||||
- **3D models as jobs** (v0.14.3). `POST /v1/3d/generations` answers with a
|
||||
job, asked after until its mesh is ready to download. ComfyUI makes it on
|
||||
the cluster's own cards, TripoSR being the tested model. The job says how
|
||||
far along it is, the model's licence is written into the GLB, and
|
||||
`per_mesh` prices it. People make them at `/3d` on the portal; `llmgw ctl
|
||||
3d` shows every owner's jobs. A model profile may also load weights of
|
||||
its own, and an admission band may wait longer than the rest, for bulk
|
||||
work.
|
||||
- **Pictures from ComfyUI** (v0.14.2). `POST /v1/images/generations` is made
|
||||
by ComfyUI, behind a gate that demands the replica's key, as an engine the
|
||||
agents install from its source checked against a digest. `per_image`
|
||||
prices it, and people make pictures at `/images` on the portal.
|
||||
- **Adapters, and the tenancy tree over the API** (v0.14.1). `llmgw ctl
|
||||
kb dataset` and `llmgw ctl train` make a LoRA adapter from a collection
|
||||
on a node's spare GPU memory, kept on the mirror and seen only by its
|
||||
owner. Organizations, teams, projects, keys and memberships are in the
|
||||
admin API and the Terraform provider. The agents' settings are edited
|
||||
with the cluster's, on the portal too, and the firewall setting reaches
|
||||
every machine.
|
||||
- **Knowledge bases** (v0.14.0). People put a company's documents in from
|
||||
their own machines with `llmgw-kb`, a separate download for Windows,
|
||||
macOS and Linux that reads PDF, Office, OpenDocument, mail and many more
|
||||
formats and sends text and vectors, never the files. Collections are kept
|
||||
apart per tenant, down to row-level security in PostgreSQL. A chat
|
||||
request, or a part of the tree by default, is answered from them with
|
||||
its sources: [docs/knowledge.md](docs/knowledge.md).
|
||||
- **One command for the whole system** (v0.13.0): `llmgw stop`, `start` and
|
||||
`restart`. Qwen3Guard is the guardrails' judge, on a GPU and on CPUs
|
||||
together.
|
||||
- **Settings that apply when changed** (v0.12.0). The cluster's
|
||||
configuration lives in the store, the same on every control plane and in
|
||||
force within seconds (`llmgw ctl config`, the console's Settings). The
|
||||
firewall can be switched on and off for the whole cluster.
|
||||
- **Security hardening** (v0.11.2): signed releases only, a token on the
|
||||
admin API, engines not run as root, TLS for the agent port, etcd and
|
||||
Patroni, a firewall for llmgw's internal ports, encrypted backups,
|
||||
bounded connections, sign-in throttling, a Content-Security-Policy.
|
||||
- **Guardrails** (v0.11.2). Policies check requests and answers for PII,
|
||||
secrets, prompt injection, deny patterns, and with a judge model; they
|
||||
observe or enforce, and attach to any part of the tenancy tree:
|
||||
[docs/guardrails.md](docs/guardrails.md).
|
||||
- **Administration** — an audit log of every change (`ctl audit`), staff
|
||||
roles (viewer, operator, billing, admin), single sign-on through OpenID
|
||||
Connect and provisioning through SCIM 2.0, key lifetime limits and
|
||||
rotation with overlap (`keys ttl`, `keys rotate`), usage export as CSV
|
||||
(`ctl usage export`).
|
||||
- **Notifications** — webhooks, signed or Slack-shaped, for a model parked,
|
||||
a node lost, a budget spent and new weights published (`notify.webhooks`,
|
||||
`ctl notify test`), and Alertmanager receiver templates.
|
||||
- **Traffic and models** — request tags, canaries of new weights with
|
||||
automatic rollback, quality suites gating updates, a response cache,
|
||||
kept and replayed requests, batches, Anthropic's Messages API, more of
|
||||
OpenAI's (structured output, images, rerank, audio, moderations),
|
||||
scaling by the clock, SLO-aware admission, TensorRT-LLM, MIG, models
|
||||
imported from Ollama, routing by what each vLLM replica's cache holds,
|
||||
LoRA adapters loaded on demand, other llmgw clusters as providers, and
|
||||
placement that knows racks, PDUs and power caps.
|
||||
- **Operations** — `server restore` and off-machine backup copies, secrets
|
||||
in Vault, forgetting a person and keeping a project's requests in the
|
||||
cluster, offline installs from a bundle, a Helm chart, tracing from the
|
||||
control plane to the engines, the admin API in OpenAPI with a Terraform
|
||||
provider, and the portal in English and Turkish with its docs at /docs.
|
||||
- **Administration** (v0.11.2):
|
||||
- an audit log of every change (`ctl audit`);
|
||||
- staff roles (viewer, operator, billing, admin);
|
||||
- single sign-on through OpenID Connect and provisioning through SCIM
|
||||
2.0;
|
||||
- key lifetime limits, and rotation with overlap (`keys ttl`, `keys
|
||||
rotate`);
|
||||
- usage export as CSV (`ctl usage export`).
|
||||
- **Notifications** (v0.11.2): webhooks, signed or Slack-shaped, for a
|
||||
model parked, a node lost, a budget spent and new weights published, and
|
||||
Alertmanager receiver templates.
|
||||
- **Traffic and models** (v0.11.2):
|
||||
- request tags, canaries of new weights with automatic rollback, and
|
||||
quality suites gating updates;
|
||||
- a response cache, kept and replayed requests, and batches;
|
||||
- Anthropic's Messages API, and more of OpenAI's (structured output,
|
||||
images, rerank, audio, moderations);
|
||||
- scaling by the clock, and SLO-aware admission;
|
||||
- TensorRT-LLM, MIG, and models imported from Ollama;
|
||||
- routing by what each vLLM replica's cache holds, and LoRA adapters
|
||||
loaded on demand;
|
||||
- other llmgw clusters as providers, and placement that knows racks,
|
||||
PDUs and power caps.
|
||||
- **Operations** (v0.11.2):
|
||||
- `server restore`, off-machine backup copies, and secrets in Vault;
|
||||
- forgetting a person, and keeping a project's requests in the cluster;
|
||||
- offline installs from a bundle, and a Helm chart;
|
||||
- tracing from the control plane to the engines;
|
||||
- the admin API in OpenAPI, with a Terraform provider;
|
||||
- the portal in English and Turkish, with its docs at /docs.
|
||||
|
||||
Every one of these is in the console as well as on the command line:
|
||||
[docs/console.md](docs/console.md).
|
||||
@@ -124,6 +186,9 @@ Every page is also in the binary: `llmgw docs`, or **Manual** in the console.
|
||||
official OpenAI and Anthropic SDKs, and llmgw's extensions
|
||||
- [docs/guardrails.md](docs/guardrails.md) — checking requests and answers:
|
||||
policies, detectors, a judge model, and where each applies
|
||||
- [docs/knowledge.md](docs/knowledge.md) — knowledge bases: collections,
|
||||
who may read them, `llmgw-kb` on people's machines, and answers from
|
||||
documents
|
||||
- [docs/choosing-models.md](docs/choosing-models.md) — which models to run,
|
||||
how many of each, and where the weights should live
|
||||
|
||||
@@ -152,7 +217,7 @@ Every page is also in the binary: `llmgw docs`, or **Manual** in the console.
|
||||
## Building
|
||||
|
||||
```sh
|
||||
make build # bin/llmgw
|
||||
make build # bin/llmgw and bin/llmgw-kb
|
||||
make test # unit tests
|
||||
make race # with the race detector
|
||||
```
|
||||
@@ -169,6 +234,7 @@ llmgw ctl cordon gpu-a-01 # stop new placement, leave what runs
|
||||
sudo llmgw doctor # on a node: what its GPUs, NVLink and fabric need
|
||||
llmgw ctl audit --since 24h # who changed what
|
||||
llmgw ctl usage export --from 2026-09-01 --to 2026-10-01 --by project,model
|
||||
llmgw ctl kb list # knowledge-base collections, grants, defaults
|
||||
```
|
||||
|
||||
Bare `llmgw` opens the console, which does all of this from one screen.
|
||||
|
||||
+911
-235
File diff suppressed because it is too large.
Load diff
@@ -28,6 +28,8 @@ message AgentMessage {
|
||||
FabricTestReply fabric_test = 6;
|
||||
UpgradeReply upgrade = 7;
|
||||
KVEvents kv_events = 8;
|
||||
TrainStatus train_status = 9;
|
||||
TrainArtifact train_artifact = 10;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,6 +74,10 @@ message ControlMessage {
|
||||
UpgradeAgent upgrade = 10;
|
||||
ControlPlanes control_planes = 11;
|
||||
Firewall firewall = 13;
|
||||
RestartAgent restart_agent = 14;
|
||||
AgentSettings agent_settings = 15;
|
||||
TrainStart train_start = 16;
|
||||
TrainCancel train_cancel = 17;
|
||||
}
|
||||
// trace is the W3C trace context (traceparent, and tracestate when there
|
||||
// is one) of the control plane's span that sent this command, so what the
|
||||
@@ -81,6 +87,12 @@ message ControlMessage {
|
||||
map<string, string> trace = 12;
|
||||
}
|
||||
|
||||
// RestartAgent has the agent start again, as llmgw restart asks of every
|
||||
// node: its engines keep serving and are taken over again.
|
||||
message RestartAgent {
|
||||
string reason = 1;
|
||||
}
|
||||
|
||||
// Firewall is whether llmgw's firewall is kept on the node — its nftables
|
||||
// table, which lets the cluster's networks alone reach the engines' ports and
|
||||
// the node exporter — sent when an agent registers and whenever the
|
||||
@@ -96,6 +108,17 @@ message Firewall {
|
||||
int32 engine_ports_to = 4;
|
||||
}
|
||||
|
||||
// AgentSettings is what the cluster's settings say of this node's agent
|
||||
// (agents.default, with agents.nodes.<id> over it), sent when it registers
|
||||
// and whenever they change. The agent keeps it beside agent.yaml, which
|
||||
// holds only what it needs to start; over that file, it is what the agent
|
||||
// runs on. A change from what it runs on has the agent start again on it,
|
||||
// its engines serving throughout.
|
||||
message AgentSettings {
|
||||
// yaml is agent.yaml's keys, as YAML; empty for none.
|
||||
string yaml = 1;
|
||||
}
|
||||
|
||||
// ControlPlanes is every control plane an agent may dial, sent by the leader
|
||||
// when an agent registers and whenever the list changes. The agent keeps it
|
||||
// beside its configuration, so a second control plane added after the node
|
||||
@@ -266,6 +289,11 @@ message RunningShard {
|
||||
// restart could not be compared with the catalog — which is exactly when the
|
||||
// catalog is most likely to have been edited.
|
||||
string profile_fp = 15;
|
||||
|
||||
// compute is where its engine said it put the model when it became ready,
|
||||
// kept by the agent across its own restarts: a control plane that takes
|
||||
// over, or restarts, learns it here.
|
||||
Compute compute = 16;
|
||||
}
|
||||
|
||||
message GPU {
|
||||
@@ -605,6 +633,18 @@ message ProvisionStatus {
|
||||
string detail = 5;
|
||||
}
|
||||
|
||||
// Compute is where an engine runs its model, as it said when it loaded it.
|
||||
message Compute {
|
||||
// backend is "CUDA", "Vulkan", "Metal" or "CPU".
|
||||
string backend = 1;
|
||||
// devices are the devices it named: "CUDA0: Quadro RTX 6000 (7.5)".
|
||||
repeated string devices = 2;
|
||||
// layers_offloaded of layers_total went to the devices; both 0 when the
|
||||
// engine did not say.
|
||||
int32 layers_offloaded = 3;
|
||||
int32 layers_total = 4;
|
||||
}
|
||||
|
||||
// ShardStatus reports a shard's lifecycle transition.
|
||||
message ShardStatus {
|
||||
string node_id = 1;
|
||||
@@ -626,6 +666,10 @@ message ShardStatus {
|
||||
// going to look.
|
||||
FetchProgress fetch = 8;
|
||||
|
||||
// compute is what the engine says it computes on, read from its own output
|
||||
// as it became ready: sent with STATE_READY when the engine says.
|
||||
Compute compute = 9;
|
||||
|
||||
enum State {
|
||||
STATE_UNSPECIFIED = 0;
|
||||
STATE_PENDING = 1;
|
||||
@@ -844,3 +888,55 @@ message LoraAdapter {
|
||||
string name = 1;
|
||||
string source = 2;
|
||||
}
|
||||
|
||||
// TrainStart has a node train a LoRA adapter (BACKLOG F4): on a base model's
|
||||
// weights from its registry, on question/answer pairs, with a Python trainer
|
||||
// the agent carries, in an environment it makes over an installed engine's.
|
||||
// One at a time per node.
|
||||
message TrainStart {
|
||||
string job_id = 1;
|
||||
// base_repo and base_revision name the base model's weights on its
|
||||
// registry (Hugging Face): safetensors, not the GGUF a catalog may serve.
|
||||
string base_repo = 2;
|
||||
string base_revision = 3;
|
||||
// data is the training pairs, NDJSON of {"q", "a"}.
|
||||
bytes data = 4;
|
||||
// params is the trainer's settings as JSON: rank, alpha, epochs, lr,
|
||||
// batch, max_len.
|
||||
string params = 5;
|
||||
// gpu_memory_mib is what the job needs of a card; the agent starts it
|
||||
// only when one has that much free, which lets it run beside an engine.
|
||||
int32 gpu_memory_mib = 6;
|
||||
// gguf asks for the adapter in llama.cpp's format too.
|
||||
bool gguf = 7;
|
||||
}
|
||||
|
||||
// TrainCancel stops a training job.
|
||||
message TrainCancel {
|
||||
string job_id = 1;
|
||||
}
|
||||
|
||||
// TrainStatus is how a training job is going, sent as it changes.
|
||||
message TrainStatus {
|
||||
string job_id = 1;
|
||||
// state is preparing, running, done or failed.
|
||||
string state = 2;
|
||||
int32 step = 3;
|
||||
int32 steps = 4;
|
||||
double loss = 5;
|
||||
// message is what is being done, or why it failed.
|
||||
string message = 6;
|
||||
// peak_mib is the most the job held of the card.
|
||||
int32 peak_mib = 7;
|
||||
}
|
||||
|
||||
// TrainArtifact is a piece of what a training job made, sent before its
|
||||
// final status: a gzipped tar of the PEFT adapter (kind peft), or the GGUF
|
||||
// adapter (kind gguf).
|
||||
message TrainArtifact {
|
||||
string job_id = 1;
|
||||
string kind = 2;
|
||||
int32 seq = 3;
|
||||
bytes data = 4;
|
||||
bool last = 5;
|
||||
}
|
||||
@@ -0,0 +1,467 @@
|
||||
// Command llmgw-kb puts a person's documents into an llmgw knowledge base
|
||||
// from their own machine (BACKLOG F2): it reads the files there, cuts them
|
||||
// into chunks, has the gateway embed them with the person's key, and sends
|
||||
// the text and vectors — only for what changed since the last run, which a
|
||||
// ledger on the machine remembers.
|
||||
//
|
||||
// Its own binary, for Windows, macOS and Linux: it carries none of the
|
||||
// server, agent or setup code that llmgw does.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"golang.org/x/term"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/kb/extract"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/kb/kbsync"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := root().Execute(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// config is what `login` keeps: the gateway, the key, and for a personal key
|
||||
// the project it acts in.
|
||||
type config struct {
|
||||
Gateway string `json:"gateway"`
|
||||
Key string `json:"key"`
|
||||
Project string `json:"project,omitempty"`
|
||||
CA string `json:"ca,omitempty"`
|
||||
}
|
||||
|
||||
func configPath() (string, error) {
|
||||
dir, err := os.UserConfigDir()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(dir, "llmgw-kb", "config.json"), nil
|
||||
}
|
||||
|
||||
// loadConfig reads the saved configuration, with the environment over it:
|
||||
// LLMGW_URL, LLMGW_API_KEY, LLMGW_PROJECT.
|
||||
func loadConfig() (config, error) {
|
||||
var c config
|
||||
if p, err := configPath(); err == nil {
|
||||
if b, err := os.ReadFile(p); err == nil {
|
||||
if err := json.Unmarshal(b, &c); err != nil {
|
||||
return c, fmt.Errorf("%s: %w", p, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for env, dst := range map[string]*string{"LLMGW_URL": &c.Gateway, "LLMGW_API_KEY": &c.Key, "LLMGW_PROJECT": &c.Project} {
|
||||
if v := strings.TrimSpace(os.Getenv(env)); v != "" {
|
||||
*dst = v
|
||||
}
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func client(flags *globalFlags) (*kbsync.Client, error) {
|
||||
c, err := loadConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if flags.gateway != "" {
|
||||
c.Gateway = flags.gateway
|
||||
}
|
||||
if flags.project != "" {
|
||||
c.Project = flags.project
|
||||
}
|
||||
if flags.ca != "" {
|
||||
c.CA = flags.ca
|
||||
}
|
||||
if c.Gateway == "" || c.Key == "" {
|
||||
return nil, errors.New("no gateway or key: run `llmgw-kb login`, or set LLMGW_URL and LLMGW_API_KEY")
|
||||
}
|
||||
h, err := kbsync.NewHTTP(c.CA)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &kbsync.Client{Base: c.Gateway, Key: c.Key, Project: c.Project, HTTP: h}, nil
|
||||
}
|
||||
|
||||
type globalFlags struct {
|
||||
gateway, project, ca, ledger string
|
||||
}
|
||||
|
||||
func (g *globalFlags) openLedger(ctx context.Context) (*kbsync.Ledger, error) {
|
||||
p := g.ledger
|
||||
if p == "" {
|
||||
var err error
|
||||
if p, err = kbsync.DefaultLedgerPath(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return kbsync.OpenLedger(ctx, p)
|
||||
}
|
||||
|
||||
func root() *cobra.Command {
|
||||
g := &globalFlags{}
|
||||
cmd := &cobra.Command{
|
||||
Use: "llmgw-kb",
|
||||
Short: "Put your documents into an llmgw knowledge base, from this machine",
|
||||
Long: `llmgw-kb reads documents on this machine, cuts them into chunks, has the
|
||||
gateway embed them with your key, and sends the text and vectors to a
|
||||
knowledge base. Files are never uploaded. Run it as often as you like: a
|
||||
ledger on this machine remembers what was sent, and only what changed is
|
||||
read and sent again.`,
|
||||
Version: version.Version,
|
||||
SilenceUsage: true,
|
||||
SilenceErrors: true,
|
||||
}
|
||||
cmd.PersistentFlags().StringVar(&g.gateway, "gateway", "", "the gateway's address (default: from login, or LLMGW_URL)")
|
||||
cmd.PersistentFlags().StringVar(&g.project, "project", "", "the project a personal key acts in (default: from login, or LLMGW_PROJECT)")
|
||||
cmd.PersistentFlags().StringVar(&g.ca, "ca", "", "a PEM file of certificates to trust besides the system's")
|
||||
cmd.PersistentFlags().StringVar(&g.ledger, "ledger", "", "the ledger file (default: in your configuration directory)")
|
||||
cmd.AddCommand(loginCmd(g), collectionsCmd(g), syncCmd(g), statusCmd(g), historyCmd(g), formatsCmd())
|
||||
return cmd
|
||||
}
|
||||
|
||||
func loginCmd(g *globalFlags) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "login",
|
||||
Short: "Save the gateway's address and your key",
|
||||
Long: `Asks for the gateway's address and your API key, checks them, and saves them
|
||||
in your configuration directory, readable by you only. The key is typed
|
||||
without being shown, so it does not end up in your shell's history.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
c, _ := loadConfig()
|
||||
in := bufio.NewReader(os.Stdin)
|
||||
ask := func(q, def string) string {
|
||||
if def != "" {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s [%s]: ", q, def)
|
||||
} else {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s: ", q)
|
||||
}
|
||||
s, _ := in.ReadString('\n')
|
||||
if s = strings.TrimSpace(s); s == "" {
|
||||
return def
|
||||
}
|
||||
return s
|
||||
}
|
||||
c.Gateway = strings.TrimRight(ask("Gateway address (https://…)", or(g.gateway, c.Gateway)), "/")
|
||||
fmt.Fprint(cmd.OutOrStdout(), "API key: ")
|
||||
var key string
|
||||
if term.IsTerminal(int(os.Stdin.Fd())) {
|
||||
b, err := term.ReadPassword(int(os.Stdin.Fd()))
|
||||
fmt.Fprintln(cmd.OutOrStdout())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key = strings.TrimSpace(string(b))
|
||||
} else {
|
||||
key, _ = in.ReadString('\n')
|
||||
key = strings.TrimSpace(key)
|
||||
}
|
||||
if key != "" {
|
||||
c.Key = key
|
||||
}
|
||||
c.Project = ask("Project, for a personal key (empty for a project's key)", or(g.project, c.Project))
|
||||
if g.ca != "" {
|
||||
c.CA = g.ca
|
||||
}
|
||||
h, err := kbsync.NewHTTP(c.CA)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cl := &kbsync.Client{Base: c.Gateway, Key: c.Key, Project: c.Project, HTTP: h, Wait: time.Second}
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
cs, err := cl.Collections(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the gateway refused: %w", err)
|
||||
}
|
||||
p, err := configPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
b, _ := json.MarshalIndent(c, "", " ")
|
||||
if err := os.WriteFile(p, b, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "Saved in %s. This key reaches %d collection(s); `llmgw-kb collections` lists them.\n", p, len(cs))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func or(a, b string) string {
|
||||
if a != "" {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func collectionsCmd(g *globalFlags) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "collections",
|
||||
Short: "List the collections your key may read, and which it may add to",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
c, err := client(g)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cs, err := c.Collections(cmd.Context())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(cs) == 0 {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "No collections. An administrator creates them and grants access (llmgw ctl kb).")
|
||||
return nil
|
||||
}
|
||||
tw := tabwriter.NewWriter(cmd.OutOrStdout(), 2, 4, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, "NAME\tACCESS\tOWNER\tEMBEDDED BY")
|
||||
for _, x := range cs {
|
||||
access := "read"
|
||||
if x.Writable {
|
||||
access = "read, write"
|
||||
}
|
||||
fmt.Fprintf(tw, "%s\t%s\t%s\t%s (%d)\n", x.Name, access, or(x.OwnerName, x.Owner), x.EmbedModel, x.Dims)
|
||||
}
|
||||
return tw.Flush()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func syncCmd(g *globalFlags) *cobra.Command {
|
||||
var o kbsync.Options
|
||||
var verbose bool
|
||||
var ocr string
|
||||
var maxMiB int64
|
||||
cmd := &cobra.Command{
|
||||
Use: "sync COLLECTION PATH...",
|
||||
Short: "Send a collection what changed in these folders and files",
|
||||
Long: `Reads the files under each PATH that the collection does not already have as
|
||||
they are, and sends their text and vectors. Run it again whenever you like;
|
||||
unchanged files are not even read.
|
||||
|
||||
Documents are named by their path under PATH, with forward slashes; with
|
||||
several PATHs, each folder's own name comes first.
|
||||
|
||||
--dry-run reads and cuts everything, sends nothing, and says what would
|
||||
leave this machine
|
||||
--delete removes documents this machine sent whose files are gone
|
||||
(only under these PATHs; nothing anybody else sent)
|
||||
--retry reads again files skipped before (after installing LibreOffice
|
||||
or Tesseract, say)
|
||||
|
||||
Without --include, every file of a known format is read (llmgw-kb formats);
|
||||
hidden files, version control folders and Office lock files are not.`,
|
||||
Args: cobra.MinimumNArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
c, err := client(g)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt)
|
||||
defer stop()
|
||||
l, err := g.openLedger(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer l.Close()
|
||||
o.Collection, o.Roots = args[0], args[1:]
|
||||
o.Extract = extract.Options{Helpers: extract.FindHelpers(), OCRLanguages: ocr, MaxBytes: maxMiB << 20}
|
||||
out := cmd.OutOrStdout()
|
||||
c.Waiting = func(why string, d time.Duration) {
|
||||
fmt.Fprintf(out, " .. waiting %s: %s\n", d, why)
|
||||
}
|
||||
start := time.Now()
|
||||
rep, err := kbsync.Sync(ctx, c, l, o, func(e kbsync.Event) {
|
||||
switch e.Kind {
|
||||
case "unchanged":
|
||||
if !verbose {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(out, " = %s\n", e.Name)
|
||||
case "sent", "would-send":
|
||||
fmt.Fprintf(out, " %-8s %s (%d chunks, %s)\n", label(e.Kind), e.Name, e.Chunks, size(e.Bytes))
|
||||
default:
|
||||
line := fmt.Sprintf(" %-8s %s", label(e.Kind), e.Name)
|
||||
if e.Detail != "" {
|
||||
line += ": " + e.Detail
|
||||
}
|
||||
fmt.Fprintln(out, line)
|
||||
}
|
||||
})
|
||||
if rep.Collection.Name != "" {
|
||||
verb := "sent"
|
||||
if o.DryRun {
|
||||
verb = "would send"
|
||||
}
|
||||
fmt.Fprintf(out, "\n%s: %d files; %s %d (%d chunks, %s), %d unchanged, %d skipped, %d failed, %d deleted, in %s\n",
|
||||
rep.Collection.Name, rep.Files, verb, rep.Sent, rep.Chunks, size(rep.Bytes), rep.Unchanged,
|
||||
rep.Skipped, rep.Failed, rep.Deleted, time.Since(start).Round(time.Second))
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rep.Failed > 0 {
|
||||
return fmt.Errorf("%d document(s) failed; run again to retry them", rep.Failed)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
f := cmd.Flags()
|
||||
f.StringSliceVar(&o.Include, "include", nil, "read only files matching these globs (\"*.pdf,docs/**/*.md\")")
|
||||
f.StringSliceVar(&o.Exclude, "exclude", nil, "never read files matching these globs")
|
||||
f.BoolVar(&o.Hidden, "hidden", false, "read hidden files and folders too")
|
||||
f.BoolVar(&o.DryRun, "dry-run", false, "send nothing; say what would be sent")
|
||||
f.BoolVar(&o.Delete, "delete", false, "remove documents this machine sent whose files are gone")
|
||||
f.BoolVar(&o.Retry, "retry", false, "read again files skipped before")
|
||||
f.IntVar(&o.Workers, "workers", 4, "files read and sent at once")
|
||||
f.StringVar(&ocr, "ocr-languages", "tur+eng", "Tesseract's languages for images and scanned pages")
|
||||
f.Int64Var(&maxMiB, "max-size", 256, "skip files larger than this many MiB")
|
||||
f.BoolVarP(&verbose, "verbose", "v", false, "list unchanged files too")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func label(kind string) string {
|
||||
switch kind {
|
||||
case "would-send":
|
||||
return "send"
|
||||
case "would-delete":
|
||||
return "delete"
|
||||
}
|
||||
return kind
|
||||
}
|
||||
|
||||
func size(b int64) string {
|
||||
switch {
|
||||
case b >= 1<<20:
|
||||
return fmt.Sprintf("%.1f MiB", float64(b)/(1<<20))
|
||||
case b >= 1<<10:
|
||||
return fmt.Sprintf("%.0f KiB", float64(b)/(1<<10))
|
||||
}
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
|
||||
func statusCmd(g *globalFlags) *cobra.Command {
|
||||
var all bool
|
||||
return &cobra.Command{
|
||||
Use: "status COLLECTION",
|
||||
Short: "Show what this machine has sent to a collection, and what it skipped",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
c, err := client(g)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m, err := c.Manifest(cmd.Context(), args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
l, err := g.openLedger(cmd.Context())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer l.Close()
|
||||
es, err := l.Entries(cmd.Context(), strings.TrimRight(c.Base, "/"), m.Collection.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
counts := map[kbsync.State]int{}
|
||||
tw := tabwriter.NewWriter(cmd.OutOrStdout(), 2, 4, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, "STATE\tDOCUMENT\tCHUNKS\tWHEN\tWHY")
|
||||
for _, e := range es {
|
||||
counts[e.State]++
|
||||
if e.State == kbsync.Sent && !all {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(tw, "%s\t%s\t%d\t%s\t%s\n", e.State, e.Name, e.Chunks, e.At.Format("2006-01-02 15:04"), e.Reason)
|
||||
}
|
||||
_ = tw.Flush()
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "\n%s holds %d documents; from this machine %d sent, %d skipped, %d failed.",
|
||||
m.Collection.Name, len(m.Documents), counts[kbsync.Sent], counts[kbsync.Skipped], counts[kbsync.Failed])
|
||||
if !all {
|
||||
fmt.Fprint(cmd.OutOrStdout(), " (--all lists the sent ones too)")
|
||||
}
|
||||
fmt.Fprintln(cmd.OutOrStdout())
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func historyCmd(g *globalFlags) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "history",
|
||||
Short: "List the latest syncs from this machine",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
l, err := g.openLedger(cmd.Context())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer l.Close()
|
||||
runs, err := l.Runs(cmd.Context(), 20)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tw := tabwriter.NewWriter(cmd.OutOrStdout(), 2, 4, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, "STARTED\tTOOK\tCOLLECTION\tSENT\tUNCHANGED\tSKIPPED\tFAILED\tDELETED")
|
||||
for _, r := range runs {
|
||||
took := "unfinished"
|
||||
if !r.Finished.IsZero() {
|
||||
took = r.Finished.Sub(r.Started).Round(time.Second).String()
|
||||
}
|
||||
fmt.Fprintf(tw, "%s\t%s\t%s\t%d\t%d\t%d\t%d\t%d\n", r.Started.Format("2006-01-02 15:04"), took,
|
||||
r.Collection, r.Sent, r.Unchanged, r.Skipped, r.Failed, r.Delete)
|
||||
}
|
||||
return tw.Flush()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func formatsCmd() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "formats",
|
||||
Short: "List the formats read, and the helpers found on this machine",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
builtIn, helper := extract.Supported()
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "Read here:\n%s\n\n", wrap(builtIn))
|
||||
fmt.Fprintf(out, "Read with a helper program:\n%s\n\n", wrap(helper))
|
||||
fmt.Fprintln(out, "Helpers:")
|
||||
for _, l := range extract.FindHelpers().Describe() {
|
||||
fmt.Fprintln(out, " "+l)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func wrap(words []string) string {
|
||||
var b strings.Builder
|
||||
line := " "
|
||||
for _, w := range words {
|
||||
if len(line)+len(w) > 78 {
|
||||
b.WriteString(strings.TrimRight(line, " ") + "\n")
|
||||
line = " "
|
||||
}
|
||||
line += w + " "
|
||||
}
|
||||
b.WriteString(strings.TrimRight(line, " "))
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The client goes onto employees' laptops: none of the server, agent, store
|
||||
// or setup code may come with it, and it must build where llmgw does not.
|
||||
func TestTheClientCarriesNoServerCode(t *testing.T) {
|
||||
out, err := exec.Command("go", "list", "-deps", ".").Output()
|
||||
if err != nil {
|
||||
t.Skipf("go list: %v", err)
|
||||
}
|
||||
for _, dep := range strings.Fields(string(out)) {
|
||||
for _, banned := range []string{"/internal/store", "/internal/router", "/internal/agent", "/internal/control",
|
||||
"/internal/cli", "/internal/guard", "github.com/jackc/pgx"} {
|
||||
if strings.Contains(dep, banned) {
|
||||
t.Errorf("llmgw-kb depends on %s", dep)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -39,7 +39,7 @@ func (d Doc) Source() string {
|
||||
// than with either reader, because the console's manual and the gateway's
|
||||
// docs portal both list them, and two lists drift.
|
||||
var docOrder = []string{"console", "quick-install", "install", "node-onboarding", "onboarding",
|
||||
"clients", "guardrails", "choosing-models", "fabrics", "facility-aware-placement", "data",
|
||||
"clients", "guardrails", "knowledge", "choosing-models", "fabrics", "facility-aware-placement", "data",
|
||||
"reference", "development"}
|
||||
|
||||
// Manual is every document, in reading order: those docOrder names, then any
|
||||
@@ -96,6 +96,12 @@ func titleOf(src, path string) string {
|
||||
//go:embed docs/admin-api.openapi.json
|
||||
var AdminOpenAPI []byte
|
||||
|
||||
// ClientOpenAPI is the client API described in OpenAPI 3.1, generated from
|
||||
// internal/clientapi's route table and served at /v1/openapi.json.
|
||||
//
|
||||
//go:embed docs/client-api.openapi.json
|
||||
var ClientOpenAPI []byte
|
||||
|
||||
// EvalSuites are the example quality suites under deploy/eval, which the
|
||||
// control plane runs by name when its eval directory has none of the name.
|
||||
//
|
||||
|
||||
+16
-7
@@ -8,8 +8,9 @@ description is.
|
||||
anything they do can be done with `curl`, a generated client or the
|
||||
[Terraform provider](../terraform-provider-llmgw/README.md). It adds and
|
||||
removes models, scales them, cordons and upgrades nodes, sets capacity
|
||||
quotas, attaches guardrail policies, and reads the fleet, usage and the audit
|
||||
log.
|
||||
quotas, attaches guardrail policies, adds organizations, teams and projects,
|
||||
issues and revokes API keys, puts people in the tree, and reads the fleet,
|
||||
usage and the audit log.
|
||||
|
||||
## Reaching it
|
||||
|
||||
@@ -99,6 +100,14 @@ curl -s "${H[@]}" -X POST $API/models/qwen-small/scale -d '{"min": 0, "max": 2,
|
||||
# A team's capacity quota.
|
||||
curl -s "${H[@]}" -X PUT $API/quotas -d '{"scope": "team:Acme/ml", "limit": {"gpus": 8}}'
|
||||
|
||||
# An organization, a team in it and a project in that; then a key for the
|
||||
# project. The answer holds the key once: only its hash is kept.
|
||||
curl -s "${H[@]}" -X POST $API/orgs -d '{"name": "Acme"}'
|
||||
curl -s "${H[@]}" -X POST $API/teams -d '{"name": "ml", "in": "Acme"}'
|
||||
curl -s "${H[@]}" -X POST $API/projects -d '{"name": "search", "in": "Acme/ml"}'
|
||||
curl -s "${H[@]}" -X POST $API/keys -d '{"name": "ci", "project": "Acme/ml/search", "expires_in": "720h"}'
|
||||
curl -s "${H[@]}" -X POST $API/users/ayse@example.com/memberships -d '{"target": "team:Acme/ml"}'
|
||||
|
||||
# Remove the model; a DELETE carries the content type too.
|
||||
curl -s "${H[@]}" -X DELETE $API/models/qwen-small
|
||||
```
|
||||
@@ -149,11 +158,11 @@ go generate ./internal/docgen
|
||||
|
||||
## What it does not cover
|
||||
|
||||
- **Organizations, teams, projects, people and API keys.** These live in the
|
||||
store and are managed with `llmgw org`, `llmgw users` and `llmgw keys` on the
|
||||
control plane, or in the web console's `/admin` pages on the client port
|
||||
behind a session. The admin API reads the tree only where a quota or a
|
||||
policy names a part of it.
|
||||
- **Making people's accounts.** People sign up, are invited, or come from
|
||||
the identity provider (OIDC, SCIM); the API lists them and puts them in
|
||||
the tree, and `llmgw users` approves, disables and deletes them. Key
|
||||
lifetime limits and spending limits are `llmgw keys ttl` and `llmgw keys
|
||||
limits`.
|
||||
- **The client-facing API** (`/v1/chat/completions` and the rest), which is
|
||||
OpenAI's and Anthropic's, on `listen`, with API keys.
|
||||
- **The rest of the admin listener**: `/metrics`, `/health`, the dashboard
|
||||
|
||||
+3060
-4
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
+69
-2
@@ -38,6 +38,14 @@ export ANTHROPIC_BASE_URL=https://llm.example.com
|
||||
export ANTHROPIC_API_KEY=$LLMGW_API_KEY
|
||||
```
|
||||
|
||||
### The API described
|
||||
|
||||
Every endpoint is described in OpenAPI 3.1 at `/v1/openapi.json`. It needs
|
||||
no key, so a client generator can be pointed at it. `/docs/api` shows it in
|
||||
Swagger UI, where *Authorize* takes a key and *Try it out* sends a request to
|
||||
the gateway. People with a part in running the gateway also find the admin
|
||||
API's document there.
|
||||
|
||||
## A first request
|
||||
|
||||
### curl
|
||||
@@ -526,6 +534,64 @@ curl https://llm.example.com/mcp \
|
||||
-d '{"jsonrpc": "2.0", "id": 1, "method": "tools/list"}'
|
||||
```
|
||||
|
||||
**Pictures, 3D models, vision and workflows.** Where the gateway has the
|
||||
models for them, the same server offers more tools — only those it has a
|
||||
model for:
|
||||
|
||||
| Tool | Does |
|
||||
|---|---|
|
||||
| `generate_image` | A picture from a prompt. `style` adds what a picture for a mesh wants: `concept` (one object, three-quarter view, grey background), `side` (an exact side view), `texture` (a seamless tile) or `none`. Answers the PNG's `file_id`, `url` and `seed`, with a 512 px preview to look at. |
|
||||
| `generate_mesh` | A GLB from a picture — `image` is a `file-…` id (a `generate_image` result, say), a data URL or base64 — or from a `prompt`, for which a concept picture is made first. `seed`, `resolution`, `texture_resolution`, and any other input the model's workflow declares in `inputs`. Answers the GLB's `file_id`, `url` and licence. |
|
||||
| `review_images` | Asks a vision model about one to four pictures (each shrunk to 1280 px on its long side); `json_schema` makes it answer that JSON. |
|
||||
| `run_workflow` | Runs a ComfyUI workflow from the inputs its catalog entry declares, and answers its result file. |
|
||||
| `job_status` | Where a mesh or a workflow run stands, and its file when done; `wait_seconds` waits for it. |
|
||||
| `gateway_status` | The picture, 3D, vision and workflow models, whether each is loaded on a GPU now, a workflow's inputs, and your last jobs. |
|
||||
|
||||
Each tool makes its request of the gateway under your key, so your
|
||||
allowlist, limits, left-out licences and budget apply, and it is charged
|
||||
and listed like any other request. What it makes is kept as your file for
|
||||
as long as job results are: fetch it from `url` with the same key (`curl -H
|
||||
"Authorization: Bearer $LLMGW_API_KEY" -o rock.glb <url>`). A tool server
|
||||
over HTTP cannot write to your disk, so an agent that needs the file
|
||||
locally — to import it in Blender — fetches it, or goes through a small
|
||||
local bridge that does.
|
||||
|
||||
A picture can be uploaded once and named by its id after: `POST /v1/files`
|
||||
with `purpose=vision` (or `user_data`) keeps it, for as long as job results
|
||||
are kept, as a file of yours that `generate_mesh`, `review_images` and a
|
||||
workflow's file inputs take as `file-…`. A call is at most 64 MB.
|
||||
|
||||
A model that is not loaded is waited for: a picture or a look at one whose
|
||||
model is starting, or waiting for another to give up the GPU, waits as a
|
||||
mesh does, saying so, rather than failing with "model loading".
|
||||
|
||||
A mesh takes from half a minute to a few minutes. A client that takes the
|
||||
answer as a stream (`Accept: text/event-stream`, as Claude Code does) is
|
||||
answered that way: `notifications/progress` for the call's
|
||||
`progressToken` while it runs — queued, waiting for the GPU, the node and
|
||||
its percent — then the result, and the call waits up to ten minutes. One
|
||||
answered in one piece waits 50 seconds; a job not done by then is answered
|
||||
with its `job_id`, to ask after with `job_status`. `wait_seconds` (up to
|
||||
900) sets either.
|
||||
|
||||
```
|
||||
claude mcp add --transport http llmgw https://llm.example.com/mcp \
|
||||
--header "Authorization: Bearer $LLMGW_API_KEY"
|
||||
```
|
||||
|
||||
### Licences you leave out
|
||||
|
||||
A model's licence is in `/v1/models` (`license`: its `id`, `name`, whether
|
||||
it allows commercial use and the regions it grants nothing in). On your
|
||||
profile page, under **Licences**, you can leave licences out: one by one,
|
||||
every licence that forbids commercial use, or every licence that grants
|
||||
nothing in a region you name (`EU` for the union). The rules cover models
|
||||
added later too. From then on those models are not listed to your keys and
|
||||
a request for one is refused with 403 and the code `licence_excluded`,
|
||||
naming the choice that left it out — on every key you own and on the
|
||||
portal's pages, whatever a key's own allowlist says. A change reaches keys
|
||||
already in use within a minute.
|
||||
|
||||
### Headers llmgw sends back
|
||||
|
||||
| Header | Says |
|
||||
@@ -545,9 +611,10 @@ its usual exception with the gateway's message in it. The ones worth knowing:
|
||||
|---|---|
|
||||
| 400 | Besides a malformed request: a prompt too long for the model, or a guardrail policy refused it (the code `content_policy_violation`). |
|
||||
| 401 | No key, a mistyped one, or one that was revoked or has expired. |
|
||||
| 403 | The key may not use that model. |
|
||||
| 403 | The key may not use that model; with the code `licence_excluded`, its licence is one you left out on your profile page. |
|
||||
| 429 | A rate limit or budget of the key, its project or its team was reached. `Retry-After` says when to try again, and is absent when waiting will not help — a spent monthly budget. |
|
||||
| 503 | No model of that name is served (the message lists the ones that are), the model is still loading, or every replica is busy and the request waited as long as it may. The SDKs retry by themselves; a name that is not served will not start to be. |
|
||||
| 503 | No model of that name is served (the message lists the ones that are), the model is still loading, or every replica is busy and the queue is full. The SDKs retry by themselves; a name that is not served will not start to be. |
|
||||
| 504 | Every replica was busy for as long as the key's band may wait. `Retry-After` says when to send it again. |
|
||||
|
||||
## Priority
|
||||
|
||||
|
||||
@@ -164,6 +164,15 @@ which): requests, tokens, errors and spend.
|
||||
team or project is guaranteed and may at most hold, and what it holds now.
|
||||
**A** adds one; on a row, change or remove it.
|
||||
|
||||
**Knowledge** lists the knowledge-base collections: each one's owner, the
|
||||
model that embeds it, how many documents and chunks it holds and who else it
|
||||
is granted to. **A** makes one. On a row: its **Documents**, **Grant…** a
|
||||
project, team, organization, person or key read or write access, **Revoke**
|
||||
a grant, **Answer from it by default…** for a part of the tree, **Try a
|
||||
search…** as an administrator, or **Delete…** it. People
|
||||
fill collections from their own machines with `llmgw-kb`
|
||||
([knowledge.md](knowledge.md)). Off on a SQLite store, and says why.
|
||||
|
||||
**Mirror** is what the model mirror holds, one row for each version of each
|
||||
model — a file, or the set of files a model of several is: shards, a vision
|
||||
projector, a config and tokenizer — with how far its download has got and
|
||||
|
||||
+95
-35
@@ -4,7 +4,7 @@ Written for the operator deciding what the gateway lets through: what a
|
||||
policy can check, what it can do, and how one is attached to a team.
|
||||
|
||||
`llmgw server init` writes a `baseline` policy that applies to every
|
||||
request, with Llama Guard 3 1B as its judge on a CPU; see
|
||||
request, with Qwen3Guard-Gen 0.6B as its judge on CPUs; see
|
||||
[install.md](install.md#2-the-control-plane) for what it checks. The
|
||||
policies are part of the cluster's settings, kept in the store: `llmgw ctl
|
||||
config edit` (or the console's Settings) changes them, and the change is in
|
||||
@@ -114,79 +114,136 @@ guard:
|
||||
|
||||
## A judge model
|
||||
|
||||
Patterns find what has a shape. For what does not — violence, self-harm,
|
||||
abuse, whatever a moderation model was trained on — a policy can have a
|
||||
**judge model on the cluster** read the text.
|
||||
Patterns find what has a shape. For what does not — violence, crime,
|
||||
sexual content, self-harm, jailbreaks, whatever a moderation model was
|
||||
trained on — a policy can have a **judge model on the cluster** read the
|
||||
text.
|
||||
|
||||
The judge must be in the catalog under the name the policy gives, as a
|
||||
judge (`role: guard`), with a replica ready. When it is not, what the
|
||||
policy judges is refused with `guard_unavailable` (`fail: closed`) or served
|
||||
unjudged (`fail: open`), and `llmgw ctl status` and the console's Overview
|
||||
say which judge and why. To serve it:
|
||||
say which judge and why.
|
||||
|
||||
### Which judge
|
||||
|
||||
| judge | runs on | size | |
|
||||
|---|---|---|---|
|
||||
| `qwen3guard-0.6b` | CPUs | 0.8 GB | the default: about 2 s a check on two threads |
|
||||
| `qwen3guard-4b` | a GPU | 2.5 GB | for a cluster with a card to spare |
|
||||
| `qwen3guard-8b` | a GPU | 4.7 GB | |
|
||||
| `llama-guard-3-1b` | CPUs | 0.9 GB | the earlier default: refuses ordinary coding requests |
|
||||
| `llama-guard-3-8b` | a GPU | 4.6 GB | |
|
||||
|
||||
Measured on the reference cluster, as the judge's probability of "unsafe"
|
||||
for ordinary coding requests against harmful ones, Llama Guard 3 1B could
|
||||
not tell them apart: "continue" 0.51, "continue the code above" 0.89, "how
|
||||
do I kill all child processes in bash" 0.92, against 0.97 to 1.00 for
|
||||
harmful requests, at any quantisation. Qwen3Guard-Gen 0.6B gave the
|
||||
ordinary ones 0.15 at the most and the harmful ones 1.00. Its verdicts are
|
||||
safe, unsafe and **controversial**; controversial counts as safe unless the
|
||||
policy says `controversial: unsafe`.
|
||||
|
||||
A policy that names one of these is enough: the leader fetches its weights
|
||||
into the mirror and serves it as a judge, and serves it again if it is
|
||||
removed from the catalog. Another judge is served by hand with `--repo`.
|
||||
|
||||
```
|
||||
llmgw ctl guard judge status # each judge the policies ask, and what is wrong with it
|
||||
llmgw ctl guard judge enable # Llama Guard 3 1B on CPUs, under the name the policies ask
|
||||
llmgw ctl guard judge enable --replace # when a model of that name is there, but not as a judge
|
||||
llmgw ctl guard judge status # each judge the policies ask, and what is wrong with it
|
||||
llmgw ctl guard judge enable # Qwen3Guard-Gen 0.6B on CPUs, or the judge the policies ask that is missing
|
||||
llmgw ctl guard judge enable --gpu # Qwen3Guard-Gen 4B on a GPU
|
||||
llmgw ctl guard judge use qwen3guard-4b qwen3guard-0.6b # the order every policy asks them in
|
||||
llmgw ctl guard judge enable --replace # when a model of that name is there, but not as a judge
|
||||
```
|
||||
|
||||
or **Do → Serve the guardrails' judge** in the console.
|
||||
|
||||
**When it refuses ordinary requests.** A refusal by a judge names the hazard
|
||||
it saw — the client is told `refused by the gateway's policy (judge:
|
||||
S6,Specialized Advice)` — and the same is kept with the request: `llmgw ctl
|
||||
request <id>`, the server's log (`guard_rule`) and the guardrails page show
|
||||
it. To see what the judge says of any text, without anything being kept:
|
||||
### A GPU and CPUs together
|
||||
|
||||
`judge.models` is a list asked in turn: the first judge that answers gives
|
||||
the verdict. The next is asked when one has no replica serving, fails, or
|
||||
does not answer within `latency_budget`; a judge with no replica at all is
|
||||
passed over at once. A judge on a GPU first and one on CPUs after it is
|
||||
the GPU's speed while it has its card, and the CPUs' when it does not.
|
||||
|
||||
```yaml
|
||||
judge:
|
||||
models: [qwen3guard-4b, qwen3guard-0.6b]
|
||||
input: block
|
||||
output: block
|
||||
latency_budget: 5s # each judge's wait
|
||||
```
|
||||
|
||||
A judge on a GPU is served as a `standard` model, on one card: it takes a
|
||||
card only from what may give one up, and waits while every card is busy,
|
||||
the judge on CPUs standing in. A judge on CPUs is `critical`, with two
|
||||
replicas, because every judged request waits on one. `llmgw ctl guard judge
|
||||
status` says, for each judge, which is asked when it gives no verdict.
|
||||
|
||||
### When it refuses ordinary requests
|
||||
|
||||
A refusal by a judge names the hazard it saw — the client is told `refused
|
||||
by the gateway's policy (judge: Violent)` — and the same is kept with the
|
||||
request: `llmgw ctl request <id>`, the server's log (`guard_rule`) and the
|
||||
guardrails page show it. To see what the judge says of any text, without
|
||||
anything being kept:
|
||||
|
||||
```
|
||||
llmgw ctl guard judge ask 'How do I sort a list in Python?'
|
||||
# llama-guard-3-1b: safe, 140 ms
|
||||
# it answered: "safe"
|
||||
# qwen3guard-0.6b: safe, 1900 ms
|
||||
# it answered: "Safety: Safe\nCategories: None"
|
||||
```
|
||||
|
||||
A judge that says `unsafe` to ordinary text is usually the wrong weights
|
||||
(another repository's or quantisation's, without Llama Guard's template:
|
||||
`judge enable --replace` serves the right ones), or a policy counting every
|
||||
hazard: give its judge `categories`, leaving out S5, S6, S8 and S13 as the
|
||||
baseline does. While finding out, `judge.input: flag` records instead of
|
||||
refusing. A Llama Guard file
|
||||
served from the mirror page is served as a judge too, under the name the
|
||||
policies ask. By hand, the catalog entry is:
|
||||
A judge that says `unsafe` to ordinary text is usually Llama Guard 3 1B
|
||||
(`llmgw ctl guard judge use qwen3guard-0.6b`), the wrong weights (another
|
||||
repository's, without the judge's template: `judge enable --replace` serves
|
||||
the right ones), or a policy counting every category: give its judge
|
||||
`categories`. The baseline counts Violent, Non-violent Illegal Acts, Sexual
|
||||
Content or Sexual Acts, Suicide & Self-Harm and Jailbreak; PII is the
|
||||
patterns', and copyright, unethical acts and politically sensitive topics
|
||||
refuse too much ordinary work. While finding out, `judge.input: flag`
|
||||
records instead of refusing.
|
||||
|
||||
A judge's file served from the mirror page — Llama Guard, Qwen3Guard,
|
||||
ShieldGemma — is served as a judge too, under the name the policies ask.
|
||||
By hand, the catalog entry is:
|
||||
|
||||
```yaml
|
||||
# a catalog model (server.yaml models:, or ctl model add)
|
||||
- name: llama-guard-3-1b
|
||||
source: http://mirror:8081/llama-guard-3-1b/…/Llama-Guard-3-1B-Q4_K_M.gguf
|
||||
- name: qwen3guard-0.6b
|
||||
source: http://mirror:8081/qwen3guard-0.6b/…/Qwen3Guard-Gen-0.6B.Q8_0.gguf
|
||||
role: guard
|
||||
guard:
|
||||
format: llamaguard3 # llamaguard3, shieldgemma or json
|
||||
categories: [Violent Crimes, Non-Violent Crimes, Sex-Related Crimes, …] # names for S1, S2, …
|
||||
format: qwen3guard # qwen3guard, llamaguard3, shieldgemma or json
|
||||
scaling: {min: 2, max: 2}
|
||||
profiles: [{name: cpu, engine: llamacpp}]
|
||||
|
||||
# in a guard policy
|
||||
judge:
|
||||
model: llama-guard-3-1b
|
||||
models: [qwen3guard-0.6b] # or model: one judge
|
||||
input: block # flag or block; empty does not ask about input
|
||||
output: flag
|
||||
categories: [S1, S10] # only these count; empty: any unsafe verdict
|
||||
latency_budget: 250ms # then the policy's fail: open serves (flagged), closed refuses
|
||||
categories: [Violent, Jailbreak] # only these count; empty: any unsafe verdict
|
||||
controversial: safe # or unsafe
|
||||
latency_budget: 250ms # each judge's wait; after the last, fail: open serves (flagged), closed refuses
|
||||
```
|
||||
|
||||
A model with `role: guard` is the guardrails' alone:
|
||||
|
||||
- it is not in `/v1/models` or the chat page, and is refused (404) to
|
||||
every key, as a name and as a fallback;
|
||||
- it is `critical` by default and must always run — on demand is refused.
|
||||
Give it two replicas on different nodes, because every judged request
|
||||
waits on it.
|
||||
- it must always run — on demand is refused. One on CPUs is `critical`
|
||||
and should have two replicas on different nodes, because every judged
|
||||
request waits on it.
|
||||
|
||||
The gateway's calls to it go past admission, accounting and the guardrails
|
||||
themselves.
|
||||
|
||||
- **Formats:**
|
||||
- `llamaguard3` reads Meta's "safe" / "unsafe
|
||||
S1,S10".
|
||||
- `qwen3guard` asks in Qwen3Guard's own words and reads "Safety: Safe |
|
||||
Unsafe | Controversial" and its "Categories:".
|
||||
- `llamaguard3` reads Meta's "safe" / "unsafe\nS1,S10"; give the catalog
|
||||
entry `categories` to have S1 recorded as its name.
|
||||
- `shieldgemma` asks Google's model whether the text breaks the listed
|
||||
principles (`categories`, each a sentence) and reads Yes or No.
|
||||
- `json` expects `{"safe": false, "categories": [...]}`.
|
||||
@@ -196,6 +253,9 @@ S1,S10".
|
||||
is read every 600 characters and at the end, so text between two
|
||||
readings has gone out already; `stream: buffer_all` holds it until it
|
||||
is read.
|
||||
- **A long text** is read by its start and its end (about 6000
|
||||
characters), so a message with a file pasted into it fits the judge's
|
||||
context rather than overflowing it and going unjudged.
|
||||
- **Cache:** the same text is not asked about twice within five minutes.
|
||||
|
||||
## Where a policy applies
|
||||
|
||||
+443
-14
@@ -129,8 +129,9 @@ That is the whole step. In order, it:
|
||||
`nvidia-smi` finds any, on CPU otherwise — and waits for it to install its
|
||||
engine
|
||||
9. closes llmgw's internal ports to everything but the cluster's networks
|
||||
10. fetches Llama Guard 3 1B, the guardrails' judge, and runs two replicas
|
||||
on the CPUs of nodes with room — this one, or the first that join
|
||||
10. fetches Qwen3Guard-Gen 0.6B, the guardrails' judge, and runs two
|
||||
replicas on the CPUs of nodes with room — this one, or the first that
|
||||
join ([guardrails.md](guardrails.md#a-judge-model) has one on a GPU too)
|
||||
11. fetches Qwen2.5 0.5B into the mirror and serves it on this node, so the
|
||||
first thing anyone opens `/chat` to find is a model that answers. A
|
||||
machine without room for it is told so within two minutes, and why.
|
||||
@@ -200,6 +201,16 @@ this by running the same command again, on the control plane that holds the
|
||||
authority: etcd is rebuilt on TLS with Patroni in maintenance mode, so
|
||||
PostgreSQL serves throughout and no failover happens.
|
||||
|
||||
Each member's PostgreSQL is sized to the machine's memory, read each time
|
||||
the command runs: a quarter of it as cache (`shared_buffers`), three quarters
|
||||
counted on for the planner, and room to sort and build indexes, all capped. A
|
||||
member that also runs a control plane or an agent gets less: about 15% as
|
||||
cache, so the gateway is not pushed into swap. A knowledge base's vector
|
||||
index is searched well only from memory, so give a store machine more RAM and
|
||||
run the command again. The new cache takes a PostgreSQL restart, which Patroni
|
||||
does one member at a time, the leader last. The leader's restart costs a few
|
||||
seconds without writes, and there is no failover.
|
||||
|
||||
Every step looks before it acts. Running it again finishes an install that
|
||||
stopped halfway and changes nothing on one that did not; it never overwrites
|
||||
`server.yaml`, the authority, or an existing account's password.
|
||||
@@ -257,8 +268,14 @@ llmgw ctl config history # who changed what, when, and why
|
||||
llmgw ctl config rollback 4 # back to version 4, as a new version
|
||||
```
|
||||
|
||||
or on the console's **Settings** screen. A change is checked before it is
|
||||
kept — one that would not load is refused — and recorded in the audit log.
|
||||
or on the console's **Settings** screen, or in a browser on the portal's
|
||||
**Settings** page (`/admin/settings`, for administrators only: the document
|
||||
can hold webhook secrets). The page edits the same document, lists its
|
||||
history, and puts an earlier version back. A change is checked before it is
|
||||
kept, so one that would not load is refused. A change made from a version
|
||||
someone else has replaced since is refused too, rather than overwriting
|
||||
theirs. Every change is recorded in the audit log, with the document's size
|
||||
but not its contents.
|
||||
Guardrails change live; anything else has every control plane start again on
|
||||
the new settings, the standbys first and the leader half a minute later, so
|
||||
one of them is always serving. Engines keep running through it.
|
||||
@@ -268,6 +285,33 @@ cluster keys — with a `baseline` guard policy, if the file had none — and
|
||||
from then on those keys in any server.yaml are not read (the server says so
|
||||
when it starts).
|
||||
|
||||
**The nodes' agents** are set the same way, under `agents`. `agents.default`
|
||||
is for every node, and `agents.nodes.<node-id>` is for one node, laid over
|
||||
the default:
|
||||
|
||||
```
|
||||
llmgw ctl config set agents.default.heartbeat_interval 5s
|
||||
llmgw ctl config set agents.nodes.cuda01.labels.gpu rtx6000
|
||||
llmgw ctl config edit # fabrics, engines, supervisor, discovery…
|
||||
```
|
||||
|
||||
Each node's `agent.yaml` then needs only what its agent must know before it
|
||||
can be told anything: `node_id`, `server` or `servers`, `hostname`,
|
||||
`mgmt_addr` and `tls`. Those keys are refused under `agents`.
|
||||
|
||||
- **Precedence.** Anything else in the file is still read, with the
|
||||
cluster's settings over it.
|
||||
- **Applying a change.** The control plane sends each node its settings, a
|
||||
second apart. An agent whose settings changed keeps them beside its
|
||||
`agent.yaml` (in `/var/lib/llmgw-agent/agent-settings.yaml`) and starts
|
||||
again on them, its engines serving throughout.
|
||||
- **Bad settings.** Settings that would not load are refused when they are
|
||||
saved. If they somehow do not load beside one node's own file, that agent
|
||||
keeps running on what it has and logs why.
|
||||
- **Editing `agent.yaml` itself.** The agent notices the change within ten
|
||||
seconds and starts again on it. If the edited file does not load, the
|
||||
agent reports it and keeps running on what it had.
|
||||
|
||||
**This machine's own keys** stay in `/etc/llmgw/server.yaml`: `listen`,
|
||||
`admin_listen`, `metrics_listen`, `agent_listen`, `agent_tls`, `auth.store`,
|
||||
`secrets`, `tracing`, `monitoring`, `mirror.api` and `mirror.tls`, the backup
|
||||
@@ -994,6 +1038,27 @@ qwen2.5-coder-3b-instruct yes 1/1 ready coder
|
||||
No restart, and it survives one: models added this way are stored, while
|
||||
anything in `server.yaml` is declarative and returns on every start.
|
||||
|
||||
A profile may load weights of its own. An embedding model, for example, can
|
||||
be served by vLLM from its repository on a card and by llama.cpp from a GGUF
|
||||
on CPUs, as one model:
|
||||
|
||||
```yaml
|
||||
name: qwen3-embed-4b
|
||||
source: Qwen/Qwen3-Embedding-4B # what a profile loads unless it says
|
||||
capabilities: { embeddings: true }
|
||||
profiles:
|
||||
- name: gpu
|
||||
engine: vllm
|
||||
gpu_class: 6000-24g
|
||||
args: ["--runner", "pooling"]
|
||||
- name: cpu
|
||||
engine: llamacpp
|
||||
source: hf:Qwen/Qwen3-Embedding-4B-GGUF/Qwen3-Embedding-4B-Q8_0.gguf
|
||||
```
|
||||
|
||||
Updates and canaries move the model's `source`; a profile with its own keeps
|
||||
it.
|
||||
|
||||
### Models that load when someone asks
|
||||
|
||||
A cluster with more models than GPUs can hold every model in its catalog and
|
||||
@@ -1226,6 +1291,161 @@ prices and budgets, rate limits, usage records, and key allowlists.
|
||||
whose allowlist leaves the external model out cannot reach it, as a fallback or
|
||||
by name. The cluster page and `ctl models` mark it `external`.
|
||||
|
||||
### Pictures from ComfyUI
|
||||
|
||||
Clients ask for pictures at OpenAI's `/v1/images/generations`. llmgw has them
|
||||
made by ComfyUI: it fills a ComfyUI workflow with the request, queues it,
|
||||
waits, and sends the pictures back as `b64_json`.
|
||||
|
||||
**On the cluster's own cards.** Use engine `comfyui`. The model is placed on a
|
||||
GPU node like any other, started when asked for, and stopped when unused.
|
||||
|
||||
**The short way: a preset.** Some models come with llmgw, with their
|
||||
workflow, custom nodes, packages and licence already found and tried:
|
||||
|
||||
```
|
||||
llmgw ctl model presets
|
||||
llmgw ctl model add flux-schnell # pictures, Apache 2.0, a card of 24 GB, Volta or newer
|
||||
llmgw ctl model add triposr # 3D models from a picture, MIT, a card of 8 GB
|
||||
```
|
||||
|
||||
A preset's profile says the card it needs, not a class, so it places on
|
||||
whatever cards the site has. When the cluster's settings name no ComfyUI,
|
||||
adding one sets `engines.comfyui` to the version this release installs, and
|
||||
every node with an NVIDIA GPU installs it; a cluster that names a version,
|
||||
or `""` for none, keeps it; the control planes then start again on that
|
||||
setting, standbys first.
|
||||
|
||||
They also come by themselves. Once a node has a card that fits one, and the
|
||||
catalog has nothing that makes what it makes, the leader adds it, on demand —
|
||||
where the cluster already installs ComfyUI; it never sets `engines.comfyui`
|
||||
by itself. A preset somebody removes stays out until it is added again.
|
||||
`llmgw ctl model presets auto off` (the setting `presets.auto`) stops it.
|
||||
|
||||
The cluster page on the portal has a **Pictures and 3D models** card under
|
||||
the models: each preset, the classes of card here it fits, whether it is in
|
||||
the catalog and who added it, why the leader has not, an **Add** button (one
|
||||
that would also install ComfyUI says so first) and the switch for adding
|
||||
them by themselves. In the console, the models screen shows the same: **A**
|
||||
adds one, **P** turns adding them by themselves on or off.
|
||||
`llmgw ctl model presets` prints it. The rest of this section is for a model
|
||||
of your own.
|
||||
|
||||
The node must offer the engine. Ask for it in the cluster's settings, and every
|
||||
node with an NVIDIA GPU installs it:
|
||||
|
||||
```
|
||||
llmgw ctl config set engines.comfyui 0.39.2
|
||||
```
|
||||
|
||||
The agent fetches ComfyUI's source from GitHub and checks it against the
|
||||
digest llmgw holds for that version. GitHub publishes none for a tag's archive,
|
||||
so a version llmgw does not know is refused. The agent then builds a Python
|
||||
environment beside the source, under `/opt/llmgw/engines/comfyui/`. Its
|
||||
dependencies are resolved as PyPI stood the day after the release. torch comes
|
||||
from PyPI (CUDA 13), from PyTorch's CUDA 12.9 build where the driver is older
|
||||
than 580, or from its CUDA 12.6 build on cards older than Turing (a V100):
|
||||
torch's CUDA 12.8 and newer builds have no kernels for them. The install is
|
||||
checked on the node's GPU, and takes about 10 GB.
|
||||
`engine_overrides` can keep it off nodes that should not have it.
|
||||
|
||||
Alternatively, install ComfyUI by hand. You need a directory with `main.py` and
|
||||
its environment in `.venv`, readable by the user engines run as
|
||||
(`llmgw-engine`). Name it in that node's `agent.yaml`; a node with one named
|
||||
there is not given the installed one:
|
||||
|
||||
```yaml
|
||||
engines:
|
||||
comfyui:
|
||||
dir: /opt/comfyui # python: defaults to /opt/comfyui/.venv/bin/python
|
||||
```
|
||||
|
||||
Then add the model:
|
||||
|
||||
```yaml
|
||||
- name: flux-schnell
|
||||
source: hf:Comfy-Org/flux1-schnell/flux1-schnell-fp8.safetensors
|
||||
comfyui:
|
||||
workflow: flux-schnell # or flux-dev, or a JSON file
|
||||
scaling: {min: 0, max: 1, idle_timeout: 15m}
|
||||
profiles:
|
||||
- name: gpu
|
||||
engine: comfyui
|
||||
gpu_class: 6000-24g
|
||||
tp: 1
|
||||
```
|
||||
|
||||
The source is fetched as for any model. It can be one file, such as the
|
||||
single-file checkpoints the built-in workflows load. It can also be a mirror
|
||||
set (a URL ending in `/`) for a model in several files, such as a diffusion
|
||||
model, its text encoders and its VAE. ComfyUI sees every file of a set by its
|
||||
own name, so a workflow of your own can load each one. A set must name the
|
||||
file that `{{checkpoint}}` stands for in `comfyui.checkpoint`. A single file
|
||||
is named by its own file name.
|
||||
|
||||
ComfyUI asks nobody for a key. The agent therefore starts it behind a gate
|
||||
(`llmgw comfyui-gate`). ComfyUI listens on the node's loopback only. The gate
|
||||
listens on the replica's port, demands the replica's key, and lets through only
|
||||
what the router asks: queueing, waiting, fetching a picture and cancelling.
|
||||
Uploads and node installs are refused. Each replica has its own directory,
|
||||
`~llmgw-engine/comfyui/REPLICA`, and ComfyUI's output can be read with `llmgw
|
||||
ctl logs NODE --source SHARD`. A profile's `args` go to `main.py`.
|
||||
`--disable-smart-memory` frees the card between pictures, at the cost of loading
|
||||
the model again for each one.
|
||||
|
||||
**On a ComfyUI server outside the cluster.** Make it an external model with
|
||||
`api: comfyui`:
|
||||
|
||||
```yaml
|
||||
- name: flux-schnell
|
||||
source: external
|
||||
external:
|
||||
url: http://10.0.0.147:8188 # ComfyUI, started with --listen
|
||||
model: flux1-schnell-fp8.safetensors # the checkpoint, as ComfyUI lists it
|
||||
api: comfyui
|
||||
workflow: flux-schnell # or flux-dev, or a JSON file
|
||||
```
|
||||
|
||||
Like any external model, its requests count as leaving the cluster.
|
||||
|
||||
`flux-schnell` (the default) and `flux-dev` are built in. They suit the
|
||||
single-file FLUX checkpoints that ComfyUI loads with `CheckpointLoaderSimple`.
|
||||
For any other model, export the workflow from ComfyUI with **Save (API)** and
|
||||
name the file. In the file, put placeholders where the request's values go:
|
||||
|
||||
- `{{prompt}}` and `{{negative_prompt}}` are the prompts.
|
||||
- `{{width}}` and `{{height}}` are the size.
|
||||
- `{{n}}` is the batch size.
|
||||
- `{{seed}}` is the seed.
|
||||
- `{{steps|4}}` and `{{guidance|3.5}}` take the value after the `|` when the
|
||||
request leaves them out.
|
||||
- `{{checkpoint}}` is the `model` above.
|
||||
|
||||
A request is OpenAI's: `prompt`, `n` (at most 4), `size` (each side a multiple
|
||||
of 16, from 256 to 2048) and `response_format: b64_json`. Beyond those, it may
|
||||
send `seed`, `steps`, `negative_prompt` and `guidance`. The seed used comes
|
||||
back in `x-llmgw-seed`. `url` is not offered, because pictures are kept
|
||||
nowhere. ComfyUI also writes the workflow into each PNG; llmgw strips it
|
||||
before the picture is sent.
|
||||
|
||||
Requests for pictures pass the same checks as any other request: keys,
|
||||
allowlists, limits, admission, and the guardrails on the prompt. A client that
|
||||
leaves takes its workflow off the ComfyUI queue.
|
||||
|
||||
Each request's usage row records how many pictures it made (`llmgw ctl request
|
||||
ID` shows them). A picture's price is set with `per_image`, in micro-units like
|
||||
the other prices, and is charged on top of the prompt's tokens:
|
||||
|
||||
```yaml
|
||||
auth:
|
||||
pricing:
|
||||
flux-schnell: {prompt_per_1k: 0, per_image: 20000}
|
||||
```
|
||||
|
||||
An external ComfyUI on a card that llmgw also places models on should be
|
||||
started with `--disable-smart-memory`. Without it, ComfyUI keeps the model on
|
||||
the card after each picture, and llmgw sees that memory as somebody else's.
|
||||
|
||||
### Federating clusters
|
||||
|
||||
Another llmgw cluster — another site, another team's GPUs — can be a provider
|
||||
@@ -1506,6 +1726,7 @@ and accounting exactly as an API call does.
|
||||
| `POST /v1/audio/transcriptions`, `/v1/audio/translations` | the multipart form passed through as it came, to an engine that serves speech to text: vLLM with Whisper |
|
||||
| `POST /v1/moderations` | answered by the guardrails' judge model (`role: guard`): `flagged`, and the judge's own categories |
|
||||
| `/v1/files`, `/v1/batches` | batches; see [Batches](#batches) |
|
||||
| `/v1/3d/generations` | 3D models, as jobs asked after; see [Jobs](#jobs-3d-models) |
|
||||
| `GET /v1/models` | the models this key may use |
|
||||
|
||||
## Watching it
|
||||
@@ -1606,6 +1827,23 @@ same event about the same thing is sent at most once an hour, so a budget that
|
||||
refuses a thousand requests is one message. `llmgw ctl notify test` sends a
|
||||
test event to each webhook and says how each answered.
|
||||
|
||||
Setup asks for them: **Notification webhooks** on its first page, or
|
||||
`--webhook` to `llmgw server init`, as often as there are webhooks:
|
||||
|
||||
```
|
||||
sudo llmgw server init --admin ops@example.com \
|
||||
--webhook https://hooks.slack.com/services/… \
|
||||
--webhook json:https://ops.example.com/llmgw
|
||||
```
|
||||
|
||||
An address of Slack's, or one with `/hooks/` in it (Mattermost, Rocket.Chat)
|
||||
or ending in `/slack`, is sent Slack's `{"text": …}`; any other the event as
|
||||
JSON; `slack:` or `json:` in front says which. They are added to the
|
||||
cluster's settings beside any already there — run again, nothing is added
|
||||
twice — and once the control plane runs on them, each is sent a test, whose
|
||||
answer setup prints. Events to choose and a signing secret are set in the
|
||||
settings afterwards (`llmgw ctl config edit`).
|
||||
|
||||
### Tracing
|
||||
|
||||
Metrics say that a model took four minutes to start; a trace says where they
|
||||
@@ -1702,9 +1940,11 @@ the file again.
|
||||
|
||||
On a machine that also runs the control plane, PostgreSQL's defaults assume it
|
||||
owns the box. A gateway swapped out to give the database cache is a worse
|
||||
outcome than a smaller cache, so cap it: `shared_buffers = 96MB` and
|
||||
`max_connections = 40` are enough for a store whose working set is measured in
|
||||
megabytes.
|
||||
outcome than a smaller cache, so size it to the machine. `server init` does this
|
||||
for a PostgreSQL it installs: `shared_buffers` about 15% of memory (at least
|
||||
96MB), `effective_cache_size` half, `work_mem` and `maintenance_work_mem`
|
||||
scaled to it, and `max_connections = 40`. A store with a large knowledge base
|
||||
wants a machine of its own, where its vector index fits in memory.
|
||||
|
||||
## Tagging requests
|
||||
|
||||
@@ -1862,6 +2102,142 @@ Each line of the file is `{"custom_id": …, "method": "POST", "url":
|
||||
or past their retention — a project under a guardrails policy with
|
||||
`no_store` is refused it.
|
||||
|
||||
## Jobs: 3D models
|
||||
|
||||
A 3D model from a picture takes minutes on a card that first has to be
|
||||
emptied of whatever else was on it. No client should hold a connection for
|
||||
that, so it is a job: queued, answered with an id, asked after, its result
|
||||
downloaded.
|
||||
|
||||
```
|
||||
curl http://gateway:8080/v1/3d/generations -H "Authorization: Bearer $LLMGW_API_KEY" \
|
||||
-d '{"model": "trellis2", "image": "<base64>"}'
|
||||
# → 202 {"id": "job_…", "status": "queued", …}
|
||||
curl http://gateway:8080/v1/3d/generations/job_… -H "Authorization: Bearer $LLMGW_API_KEY"
|
||||
# → {"status": "completed", "output_file_id": "file-…", "content_type": "model/gltf-binary", "usage": {…}}
|
||||
curl http://gateway:8080/v1/files/file-…/content -H "Authorization: Bearer $LLMGW_API_KEY" -o model.glb
|
||||
curl -X POST http://gateway:8080/v1/3d/generations/job_…/cancel -H "Authorization: Bearer $LLMGW_API_KEY"
|
||||
```
|
||||
|
||||
- **Status:** `queued`, `running`, `completed`, `failed` (with `error`),
|
||||
`cancelling`, `cancelled`, `expired` (not finished within 24 h).
|
||||
`progress` carries what the engine said of the work so far, once it says
|
||||
anything; `GET /v1/3d/generations` lists the caller's jobs. While the
|
||||
model waits for another on-demand model to give up the card,
|
||||
`progress` is `{"state":"waiting_gpu","held_by":"flux-schnell","eta_seconds":N}`:
|
||||
N is when the holder is stopped (idle and old enough to evict, or at its
|
||||
idle timeout) plus how long the model's last start took, and is left out
|
||||
when that is not known. A request made directly gets the same as a 503
|
||||
with `X-Llmgw-Waiting-For`, `X-Llmgw-Eta-Seconds` and `Retry-After`.
|
||||
- **Refused at the door:** no key, a model that is not there or not in the
|
||||
key's allowlist, a request the model cannot serve. Refused when served,
|
||||
and said as the job's error: a spent budget, the guardrails, a model
|
||||
that will not come up.
|
||||
- **How it is served:** the request is kept as it was sent and replayed by
|
||||
whichever gateway holds the job's lease, through the same path a
|
||||
client's request takes — limits, admission in the key's band, guardrails,
|
||||
accounting — by the leading control plane, since a standby has nothing
|
||||
to send it to. A model said to fit nowhere is asked for again for five
|
||||
minutes before that is the job's error: an agent re-registering says so
|
||||
for a few seconds. It is one usage record under the job's id, tagged
|
||||
`job=<id>`, charged like any request; `GET` shows the usage once it is
|
||||
done. A model still loading, a rate limit or a full queue is waited out
|
||||
for as long as the job has: nobody is holding a connection, which is the
|
||||
point.
|
||||
- **Surviving a restart:** a gateway that stops hands its jobs to another
|
||||
within two minutes; a job it was running is run again from the start.
|
||||
- **What it keeps:** the request as sent, and the result as a file owned
|
||||
by the key's project, kept `router.batch.retain_days` (30) like a batch's
|
||||
files; the job's record goes with it. A project under a `no_store`
|
||||
policy is refused jobs, as it is batches.
|
||||
|
||||
A request may also send `seed` and `resolution` (the model's own grid, a
|
||||
multiple of 32 from 64 to 2048). The picture should show one object on a
|
||||
plain background: the model makes a mesh of what it sees, background
|
||||
included.
|
||||
|
||||
### A model that makes them
|
||||
|
||||
A 3D model is made by a ComfyUI model whose `comfyui.output` is `meshes`
|
||||
(see [Pictures from ComfyUI](#pictures-from-comfyui) for engine `comfyui`).
|
||||
Its workflow is a JSON file in ComfyUI's API format: `{{image}}` is the
|
||||
uploaded picture, which a `LoadImage` node reads; `{{seed}}`,
|
||||
`{{resolution|256}}` and `{{checkpoint}}` are as for pictures. The node that
|
||||
saves the mesh may save a GLB or an OBJ; an OBJ leaves the gateway as a GLB,
|
||||
vertex colours kept.
|
||||
|
||||
TripoSR (MIT), on any card of 8 GB, comes with llmgw: `llmgw ctl model add
|
||||
triposr`. What that adds is this entry, and its workflow is built in:
|
||||
|
||||
```yaml
|
||||
- name: triposr
|
||||
source: hf:stabilityai/TripoSR/model.ckpt
|
||||
comfyui:
|
||||
workflow: triposr # built in; a model of your own names a JSON file on every control plane
|
||||
output: meshes
|
||||
nodes:
|
||||
- repo: https://github.com/flowtyone/ComfyUI-Flowty-TripoSR
|
||||
commit: a0c94ac60a7cc062604f61aeeea6d0d493521de3
|
||||
requirements: [omegaconf, trimesh, scikit-image, imageio,
|
||||
"transformers>=4.50.3,<5", "huggingface-hub<1", tokenizers,
|
||||
rembg, onnxruntime]
|
||||
license:
|
||||
id: MIT
|
||||
name: MIT
|
||||
commercial: true
|
||||
url: https://huggingface.co/stabilityai/TripoSR/blob/main/LICENSE
|
||||
notice: "Made with TripoSR (Stability AI and Tripo AI, MIT). ..."
|
||||
scaling: {min: 0, max: 1, idle_timeout: 10m}
|
||||
profiles:
|
||||
- {name: gpu, engine: comfyui, tp: 1, gpu: {min_compute_capability: 7.0, min_memory_gb: 8}}
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"1": {"class_type": "LoadImage", "inputs": {"image": "{{image}}"}},
|
||||
"5": {"class_type": "LLMGWPrepareForMesh", "inputs": {"image": ["1", 0], "mask": ["1", 1],
|
||||
"foreground_ratio": 0.85, "background": 0.5, "remove_background": "auto", "rembg_model": "u2net"}},
|
||||
"2": {"class_type": "TripoSRModelLoader", "inputs": {"model": "{{checkpoint}}", "chunk_size": 8192}},
|
||||
"3": {"class_type": "TripoSRSampler", "inputs": {"model": ["2", 0], "reference_image": ["5", 0],
|
||||
"geometry_resolution": "{{resolution|256}}", "threshold": 25.0}},
|
||||
"4": {"class_type": "TripoSRViewer", "inputs": {"mesh": ["3", 0]}}
|
||||
}
|
||||
```
|
||||
|
||||
`LLMGWPrepareForMesh` comes with the gate: it cuts the object out — by the
|
||||
picture's alpha, or by rembg where it has none — and centres it on a plain
|
||||
background, without which the background becomes geometry.
|
||||
|
||||
- **Custom nodes** are named by https repository and full commit (a branch
|
||||
is refused: it moves). The gate puts each in the replica's own
|
||||
`custom_nodes` and installs its `requirements` beside the replica, never
|
||||
into ComfyUI's environment. A package named there is installed at the
|
||||
range named; every other one at the version ComfyUI already runs. A
|
||||
package named is first on ComfyUI's path, so its range must suit ComfyUI
|
||||
as well. TripoSR's checkpoint loads with transformers 4, which ComfyUI's
|
||||
`>=4.50.3` allows.
|
||||
- **Progress:** the gate listens to ComfyUI's websocket and the router
|
||||
writes what it hears into the job: `{"state": "queued", "ahead": 2}`,
|
||||
then `{"state": "running", "node_class": "KSampler", "step": 12,
|
||||
"steps": 30, "percent": 40}` for a node that counts its steps, and
|
||||
`done` at the end.
|
||||
- **Licence:** a model's `license` is shown in `/v1/models`, in the job and
|
||||
on the portal, and written into the GLB's `asset.extras.license`, since
|
||||
the file travels without the API. Choose the model for its licence before
|
||||
its quality when what it makes is shipped. TRELLIS.2 needs Meta's gated
|
||||
DINOv3 and, as published, a background remover under a non-commercial
|
||||
licence; Hunyuan3D's licence excludes the EU, the UK and South Korea.
|
||||
- **Price:** `per_mesh` in `auth.pricing`, beside `per_image`; each usage
|
||||
row says how many meshes it made (`meshes`).
|
||||
- **Cards:** the strongest models today (TRELLIS.2, Direct3D-S2) need
|
||||
flash-attention, which runs on Ampere and newer. A Turing card such as
|
||||
the RTX 6000 runs TripoSR.
|
||||
|
||||
People make them at `/3d` on the portal: they choose a picture and watch
|
||||
the job, and the mesh is shown in a viewer (model-viewer, served from the
|
||||
gateway) with a download link and the licence. `llmgw ctl 3d list`, `get`
|
||||
and `cancel` show and stop every owner's jobs.
|
||||
|
||||
## Conversations across models
|
||||
|
||||
Off everywhere until a project asks. llmgw otherwise stores no message content
|
||||
@@ -2150,13 +2526,29 @@ llmgw ctl config set firewall.enabled false # every machine's table taken out
|
||||
llmgw ctl config set firewall.enabled true # and put back as it was
|
||||
```
|
||||
|
||||
It is in force within seconds: each node's agent takes its table out or puts
|
||||
it back, and each control plane's `llmgw-firewall-sync.path` (installed by
|
||||
`server init` and `server add`; `sudo llmgw firewall sync --install` on one
|
||||
set up before) does the same there. The rules files stay, so turning it back
|
||||
on restores them. Nodes added while it is off get none. A separate mirror
|
||||
machine and the store's members keep theirs for now. `firewall_networks`
|
||||
changes reach the nodes the same way.
|
||||
It is in force on every machine within a minute, each in its own way:
|
||||
|
||||
- **nodes**: the agent takes its table out or puts it back;
|
||||
- **control planes**: `llmgw-firewall-sync.path`, installed by `server
|
||||
init` and `server add` (`sudo llmgw firewall sync --install` on one set up
|
||||
before);
|
||||
- **a separate mirror machine**: the leading control plane tells the mirror,
|
||||
which writes it to `.firewall.json` in its directory, and the same path
|
||||
unit applies it. `mirror init` installs it; on a mirror set up before, run
|
||||
`sudo llmgw mirror init` again with the same flags;
|
||||
- **store members that are nothing else**: they run no llmgw, so
|
||||
`llmgw-firewall-follow.timer` reads what the leading control plane keeps in
|
||||
the store, from the member's own copy, once a minute. `server init
|
||||
--patroni` installs it; run it again on a cluster set up before.
|
||||
|
||||
The rules files stay, so turning it back on restores them. Nodes added while
|
||||
it is off get none. Networks added to `firewall_networks` reach every machine
|
||||
the same way. One taken out of it stays let in until it is taken out of
|
||||
`/etc/llmgw/firewall.nft` by hand: the rules are only ever added to.
|
||||
|
||||
The leading control plane also tells the mirror machine where every node and
|
||||
control plane is, so a node on a network the mirror had not let in can fetch
|
||||
its weights without a `firewall_networks` entry.
|
||||
|
||||
Only those ports are touched. The table's policy is to accept, so whatever
|
||||
else the machine serves — and whatever firewall it already has — is left as
|
||||
@@ -2294,6 +2686,43 @@ A cluster set up online becomes offline the same way: `bundle import` on the
|
||||
mirror and control plane, then the `offline:` block in `server.yaml` and a
|
||||
restart.
|
||||
|
||||
## Stopping and starting the system
|
||||
|
||||
From a control plane:
|
||||
|
||||
```
|
||||
llmgw status # does it serve; its control planes, nodes, models, store
|
||||
llmgw stop --note "maintenance" # every model taken down, every request answered 503
|
||||
llmgw start # every model back as its catalog entry asks
|
||||
sudo llmgw restart # every agent and control plane started again, engines serving throughout
|
||||
```
|
||||
|
||||
`llmgw stop` frees every GPU and answers requests for a model with 503
|
||||
(`system_stopped`, with Retry-After) until `llmgw start`. The control planes,
|
||||
agents and mirror keep running, so the console, the admin pages and `llmgw
|
||||
start` keep working, and no model's definition changes. It is the cluster's
|
||||
setting `serving.stopped`, so a restart does not undo it, and it is in the
|
||||
audit with its note. `--wait` waits until every replica is down, or every
|
||||
model has its replicas again.
|
||||
|
||||
`llmgw restart` restarts every node's agent, then the other control planes
|
||||
over ssh (`--ssh-user`, the one sudo was run by), then this one — the leader
|
||||
last, so it hands over to a standby already restarted. Engines keep serving
|
||||
and are taken over again; nothing is placed anew.
|
||||
|
||||
One machine's own services are `llmgw service`:
|
||||
|
||||
```
|
||||
sudo llmgw service status
|
||||
sudo llmgw service stop # agent, control plane, mirror — whichever this machine has
|
||||
sudo llmgw service start
|
||||
sudo llmgw service restart agent # or name one: server, agent, mirror
|
||||
```
|
||||
|
||||
Stopping a node's agent leaves its engines serving — the agent takes them
|
||||
over again when it starts — unless `llmgw service stop --engines` stops them
|
||||
too, and the control plane then places their replicas elsewhere.
|
||||
|
||||
## Removing llmgw from a machine
|
||||
|
||||
```
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
# Knowledge bases
|
||||
|
||||
A knowledge base is a collection of a company's documents that the cluster
|
||||
can answer from. People put their documents into it from their own machines
|
||||
with `llmgw-kb`. The tool reads the files on that machine, cuts them into
|
||||
passages ("chunks"), has the gateway embed each passage with the person's own
|
||||
key, and sends the text and the vectors. The files themselves never leave
|
||||
the machine.
|
||||
|
||||
## Why retrieval, and not training
|
||||
|
||||
Retrieval is the tool for *knowledge*: policies, procedures, contracts. The
|
||||
answer can name its source, a changed document is sent again in minutes, and
|
||||
a passage only ever reaches people allowed to read its collection.
|
||||
|
||||
Training a model on the documents (LoRA) changes how it *writes*, not what it
|
||||
reliably *knows*. It has no access control: whatever goes into the weights
|
||||
is available to everyone who may use them.
|
||||
|
||||
## What it needs
|
||||
|
||||
- **The PostgreSQL store with pgvector.** On a SQLite store knowledge bases
|
||||
are off, and `llmgw ctl kb list` and the console say why.
|
||||
- `llmgw server init` and the high-availability database setup install
|
||||
pgvector and enable it.
|
||||
- A cluster set up before this build needs it by hand, once:
|
||||
1. Install the package on every database machine:
|
||||
`sudo apt-get install postgresql-16-pgvector` (use the server's
|
||||
version number).
|
||||
2. On the leader, enable it in llmgw's database:
|
||||
`sudo -u postgres psql -d llmgw -c 'CREATE EXTENSION vector'`.
|
||||
3. Restart the control planes: `llmgw restart`.
|
||||
- **An embedding model in the catalog**, with `capabilities.embeddings`.
|
||||
Two good choices handle Turkish and run on CPUs: `BAAI/bge-m3` (1024
|
||||
dimensions) and `Qwen/Qwen3-Embedding-0.6B`.
|
||||
|
||||
Add it as any model is added (**Serve a model** in the console, or the
|
||||
catalog), with its entry saying that it embeds:
|
||||
|
||||
```yaml
|
||||
capabilities:
|
||||
embeddings: true
|
||||
```
|
||||
|
||||
A collection stays with the model it was made with: vectors from two
|
||||
models cannot be compared.
|
||||
|
||||
Some embedding models are trained with an instruction before the
|
||||
question and none before the documents. Give it as the entry's
|
||||
`query_prefix`; llmgw puts it before questions only. For
|
||||
Qwen3-Embedding:
|
||||
|
||||
```yaml
|
||||
query_prefix: "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery: "
|
||||
```
|
||||
|
||||
With pgvector 0.7 or later a collection's index is kept at half
|
||||
precision (`halfvec`), which halves its memory and takes models of up to
|
||||
4000 dimensions; with an older pgvector the limit is 2000. A collection
|
||||
keeps the kind it was made with.
|
||||
|
||||
## For administrators
|
||||
|
||||
### Make a collection
|
||||
|
||||
A collection belongs to one part of the tenancy tree: an organization, a
|
||||
team, a project or a person. Whoever is in the owner, at any level below it,
|
||||
may read it and add to it.
|
||||
|
||||
```sh
|
||||
llmgw ctl kb create team:Acme/hr policies bge-m3
|
||||
llmgw ctl kb create user:USER-ID notes bge-m3 --org Acme
|
||||
```
|
||||
|
||||
The model's dimension is asked of the model itself, so the model has to be
|
||||
able to serve when the collection is made. `--dims 1024` gives it instead.
|
||||
|
||||
In the console: **Knowledge**, then **A**.
|
||||
|
||||
### Who may read and add
|
||||
|
||||
The owner, and everything below it, may read and add. A **grant** gives the
|
||||
same to another part of the same organization, to a person, or to one key:
|
||||
|
||||
```sh
|
||||
llmgw ctl kb grant Acme/policies project:Acme/sales/crm # read
|
||||
llmgw ctl kb grant Acme/policies team:Acme/legal --write # read and add
|
||||
llmgw ctl kb revoke Acme/policies project:Acme/sales/crm
|
||||
```
|
||||
|
||||
How access works:
|
||||
|
||||
- **Grants add up down the tree.** A grant to a team reaches its projects
|
||||
and their keys.
|
||||
- **Nothing crosses organizations.** A grant to another organization's
|
||||
team, project, key or person is refused.
|
||||
- **A person's own key** (a key with no project) reads nothing by itself.
|
||||
The person names the project they act in (`llmgw-kb login` asks for it,
|
||||
or `--project`) and must be a member of it. Joining a second team
|
||||
therefore does not quietly widen what every key of theirs can read.
|
||||
- **No access looks like no collection.** A collection a key may not read
|
||||
is answered exactly as one that does not exist, so even its name does not
|
||||
leak.
|
||||
|
||||
### Answer from collections by default
|
||||
|
||||
Clients that cannot add a header still get answers from the documents when
|
||||
their part of the tree has a default:
|
||||
|
||||
```sh
|
||||
llmgw ctl kb default team:Acme/hr Acme/policies Acme/forms
|
||||
llmgw ctl kb default key:KEY-ID Acme/policies
|
||||
llmgw ctl kb default team:Acme/hr --clear
|
||||
```
|
||||
|
||||
- **Which default applies.** The narrowest one: a key's over its project's,
|
||||
a project's over its team's, a team's over its organization's.
|
||||
- **A request can override it.** A request that names collections uses
|
||||
those instead, and `X-Llmgw-Knowledge: none` turns the default off for
|
||||
that request.
|
||||
- **A default never refuses a request.** A collection the caller may not
|
||||
read is left out for them. If the search fails (the embedding model is
|
||||
down, say), the request is answered without passages and the failure is
|
||||
logged.
|
||||
- **Changes take effect within 30 seconds.**
|
||||
|
||||
In the console: **Knowledge**, Enter on a collection, then **Answer from it
|
||||
by default…**.
|
||||
|
||||
### Look after it
|
||||
|
||||
```sh
|
||||
llmgw ctl kb list # collections, owners, documents, grants
|
||||
llmgw ctl kb docs Acme/policies # its documents, who added each and when
|
||||
llmgw ctl kb rm Acme/policies old/form.pdf # one document out
|
||||
llmgw ctl kb search Acme/policies how many days of annual leave
|
||||
llmgw ctl kb delete Acme/policies # the collection, its documents and grants
|
||||
```
|
||||
|
||||
Rules that apply:
|
||||
|
||||
- A project, team or organization that owns a collection is not deleted
|
||||
until its collections are. This is the same rule as for one with members.
|
||||
- A person's own collections go with their account (`users delete`,
|
||||
`users forget`).
|
||||
- Every document added or removed is in the audit log (`llmgw ctl audit`),
|
||||
by name. The text is never recorded.
|
||||
|
||||
### A large import
|
||||
|
||||
A few thousand documents ask the embedding model for far more than it can
|
||||
answer at once. Give the key that sends them a band of its own, below the
|
||||
people's, that may wait for room for minutes rather than the 30 seconds the
|
||||
others wait:
|
||||
|
||||
```sh
|
||||
llmgw ctl config set router.admission.bands '[{name: gold, priority: 300, saturation_ceiling: 0.85},
|
||||
{name: bronze, priority: 100, saturation_ceiling: 0.3}, {name: bulk, priority: 50, saturation_ceiling: 0.3, queue_ttl: 5m}]'
|
||||
llmgw keys create import --project Acme/docs --band bulk
|
||||
```
|
||||
|
||||
The people's questions then go first, and the import waits behind them. A
|
||||
request that waits as long as its band allows is answered 504 with
|
||||
`Retry-After`, and `llmgw-kb` sends it again rather than counting the
|
||||
passage as failed. It gives up after 15 minutes.
|
||||
|
||||
## For everyone: `llmgw-kb`
|
||||
|
||||
### Get it
|
||||
|
||||
Download `llmgw-kb` for your system from the release page, next to `llmgw`:
|
||||
|
||||
| System | File |
|
||||
|---|---|
|
||||
| Windows (Intel/AMD) | `llmgw-kb-VERSION-windows-amd64.exe` |
|
||||
| Windows (ARM) | `llmgw-kb-VERSION-windows-arm64.exe` |
|
||||
| macOS (Apple silicon) | `llmgw-kb-VERSION-darwin-arm64` |
|
||||
| macOS (Intel) | `llmgw-kb-VERSION-darwin-amd64` |
|
||||
| Linux | `llmgw-kb-VERSION-linux-amd64` / `-arm64` |
|
||||
|
||||
Rename it to `llmgw-kb` (`llmgw-kb.exe` on Windows) and put it somewhere on
|
||||
your PATH. You do not need to install anything else.
|
||||
|
||||
On macOS, a downloaded program that is not signed is stopped the first time.
|
||||
Either right-click it and choose Open, or run
|
||||
`xattr -d com.apple.quarantine llmgw-kb`.
|
||||
|
||||
### Log in once
|
||||
|
||||
```sh
|
||||
llmgw-kb login
|
||||
```
|
||||
|
||||
It asks for three things, checks them, and saves them where only you can
|
||||
read them:
|
||||
|
||||
- **The gateway's address.**
|
||||
- **Your API key.** It is typed without being shown.
|
||||
- **A project.** Needed only for a personal key.
|
||||
|
||||
Instead of logging in you can set `LLMGW_URL`, `LLMGW_API_KEY` and
|
||||
`LLMGW_PROJECT`.
|
||||
|
||||
```sh
|
||||
llmgw-kb collections # what your key may read, and which you may add to
|
||||
```
|
||||
|
||||
### Send your documents
|
||||
|
||||
```sh
|
||||
llmgw-kb sync policies ~/Documents/HR
|
||||
llmgw-kb sync policies "C:\Users\ayse\Documents\İK" --include "*.pdf,*.docx"
|
||||
llmgw-kb sync policies ./handbook ./forms --exclude "drafts/**"
|
||||
```
|
||||
|
||||
Run it as often as you like. The first run sends everything. Later runs
|
||||
send only what changed, and do not even read files whose size and time have
|
||||
not changed.
|
||||
|
||||
| Option | What it does |
|
||||
|---|---|
|
||||
| `--dry-run` | Reads and cuts everything, sends nothing, and lists what would leave the machine and how much. |
|
||||
| `--delete` | Also removes documents whose files are gone. Only those this machine sent, from these folders; never anything someone else sent. |
|
||||
| `--retry` | Reads again files skipped before, for example after installing LibreOffice. |
|
||||
| `--include`, `--exclude` | Glob patterns. `*.pdf` matches a file name; `reports/**/*.xlsx` matches a path. |
|
||||
| `--hidden` | Reads hidden files and folders too (left out by default). |
|
||||
| `-v` | Lists unchanged files too. |
|
||||
|
||||
Documents are named by their path under the folder, with forward slashes,
|
||||
on every system. With several folders, each folder's own name comes first.
|
||||
|
||||
```sh
|
||||
llmgw-kb status policies # what this machine sent and skipped, and why
|
||||
llmgw-kb history # the latest runs
|
||||
llmgw-kb formats # what can be read here
|
||||
```
|
||||
|
||||
### What it reads
|
||||
|
||||
| Read here | Formats |
|
||||
|---|---|
|
||||
| Documents | PDF (text), Word (`.docx`), Excel (`.xlsx`), PowerPoint (`.pptx`, with speaker notes), OpenDocument (`.odt`, `.ods`, `.odp`), RTF, EPUB |
|
||||
| Web and mail | HTML, MHTML, e-mail (`.eml`, with its attachments) |
|
||||
| Text | Markdown, plain text, CSV, JSON, YAML, XML, subtitles, notebooks (`.ipynb`), source code |
|
||||
| Archives | ZIP, with every file inside it read in turn |
|
||||
|
||||
| Read with a helper on your machine | Helper |
|
||||
|---|---|
|
||||
| Word, Excel and PowerPoint 97–2003 (`.doc`, `.xls`, `.ppt`), Pages, Numbers, Keynote and others | [LibreOffice](https://www.libreoffice.org) |
|
||||
| Images and scanned PDFs | [Tesseract](https://tesseract-ocr.github.io) (with the Turkish language data), plus Poppler's `pdftoppm` for PDFs |
|
||||
| PDFs the built-in reader cannot parse | Poppler's `pdftotext` |
|
||||
|
||||
`llmgw-kb formats` says which helpers it found. Text in UTF-8, UTF-16, or
|
||||
the Windows Turkish and Western code pages is read correctly.
|
||||
|
||||
A file that cannot be read is skipped with the reason, and the run goes on.
|
||||
Reasons include a locked Office file, a format with no helper, and a
|
||||
scanned page with no OCR.
|
||||
|
||||
### What leaves your machine
|
||||
|
||||
For each new or changed document, two things leave the machine, both over
|
||||
the gateway's TLS connection:
|
||||
|
||||
- **The passages, to be embedded.** They go through `/v1/embeddings` like
|
||||
any request of yours. Your key's limits, budget and guardrails apply.
|
||||
- **The document itself.** Its name, its file's SHA-256 digest and size,
|
||||
and the passages with their vectors.
|
||||
|
||||
The file itself is never sent. `--dry-run` shows exactly what would be.
|
||||
|
||||
### The ledger
|
||||
|
||||
`llmgw-kb` keeps a small SQLite database of what it did with every
|
||||
document: the file it came from, its size, time and digest, and whether it
|
||||
was sent, skipped or failed, with the reason.
|
||||
|
||||
| System | Where it is |
|
||||
|---|---|
|
||||
| Windows | `%AppData%\llmgw-kb\ledger.db` |
|
||||
| macOS | `~/Library/Application Support/llmgw-kb/ledger.db` |
|
||||
| Linux | `~/.config/llmgw-kb/ledger.db` |
|
||||
|
||||
Deleting the ledger is safe: the next run hashes every file once, finds what
|
||||
the collection already has, and sends nothing that is unchanged. `--ledger`
|
||||
uses another file.
|
||||
|
||||
## Asking questions of a collection
|
||||
|
||||
A chat request names the collections to answer from in a header. Any client
|
||||
that can add a header can do it: the OpenAI and Anthropic SDKs, Continue, or
|
||||
curl.
|
||||
|
||||
```sh
|
||||
curl https://llm.example.com/v1/chat/completions \
|
||||
-H "Authorization: Bearer $LLMGW_API_KEY" \
|
||||
-H "X-Llmgw-Knowledge: politikalar" \
|
||||
-d '{"model": "qwen2.5-coder-14b-instruct",
|
||||
"messages": [{"role": "user", "content": "Kaç gün yıllık iznim var?"}]}'
|
||||
```
|
||||
|
||||
```python
|
||||
client.chat.completions.create(
|
||||
model="qwen2.5-coder-14b-instruct",
|
||||
messages=[{"role": "user", "content": "Kaç gün yıllık iznim var?"}],
|
||||
extra_headers={"X-Llmgw-Knowledge": "politikalar"},
|
||||
)
|
||||
```
|
||||
|
||||
What the gateway does with such a request:
|
||||
|
||||
1. **It finds the collections.** It looks up the ones named, by name or ID,
|
||||
among those the key may read. A collection it may not read is answered
|
||||
404, the same as one that does not exist.
|
||||
2. **It embeds the question.** The last user message is embedded by the
|
||||
collection's model, on the caller's own account: their limits, budget and
|
||||
guardrails apply.
|
||||
3. **It searches.** The collections are searched by meaning and by words
|
||||
(below).
|
||||
4. **It adds the passages.** The best passages are numbered and added to the
|
||||
system message, with an instruction to answer from them, cite them as
|
||||
[1], [2], and say so when they do not hold the answer. The system message
|
||||
is created if the request has none. Passages that would not fit the
|
||||
model's context are left out, the weakest first.
|
||||
5. **It judges and sends the request.** The request, passages included, then
|
||||
goes through the guardrails and to the model as usual. A document
|
||||
carrying instructions aimed at its reader is judged like anything else
|
||||
the model reads.
|
||||
|
||||
On the portal's chat page, **Documents** picks a collection the person may
|
||||
read in the chosen project, the default, or none. Under each answer, the
|
||||
sources it was given are listed by number.
|
||||
|
||||
How it is shaped:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| `X-Llmgw-Knowledge` | Collections, comma separated; `none` for no passages, whatever the default. |
|
||||
| `X-Llmgw-Knowledge-K` | How many passages, 1 to 20; 5 by default. |
|
||||
| `X-Llmgw-Project` | For a personal key: the project it acts in. |
|
||||
| `x-llmgw-sources` (answer) | The passages given, as `document#chunk`, in their numbered order; empty when nothing matched. |
|
||||
|
||||
Behaviour to know about:
|
||||
|
||||
- **No passages matched.** The model is told that nothing in the documents
|
||||
answers the question, and asked to say so rather than answer from general
|
||||
knowledge.
|
||||
- **Other APIs.** Requests through `/v1/responses` and `/v1/messages`
|
||||
(Codex, Claude Code) work the same way, since they are answered as chat
|
||||
completions.
|
||||
- **Separate prefix caches.** On vLLM replicas, a request given passages
|
||||
carries a `cache_salt` derived from the collections they came from.
|
||||
Requests that could be given the same passages still share cached blocks,
|
||||
and no one else's do: not another organization's, nor another team's of
|
||||
the same one. How fast an answer starts therefore says nothing about what
|
||||
someone else asked.
|
||||
- **No answer cache.** Answers from knowledge bases are never served from
|
||||
the answer cache, because two keys of one project can read different
|
||||
collections.
|
||||
|
||||
### How a passage is chosen
|
||||
|
||||
Each passage scores the following, and the highest scores win:
|
||||
|
||||
- its similarity in meaning to the question (cosine, 0 to 1), plus
|
||||
- up to 0.3 for the share of the question's words it contains.
|
||||
|
||||
Words almost every passage has ("ne", "için", "the", "of") are not counted.
|
||||
The meaning does most of the work: "how much leave do fathers get" finds the
|
||||
paternity page, not the annual-leave page that shares more of its words.
|
||||
The word match helps with what meaning cannot carry, such as a form number,
|
||||
a product code or a name.
|
||||
|
||||
Metrics: `llmgw_kb_retrievals_total{result}` (found, nothing, refused) and
|
||||
`llmgw_kb_retrieval_seconds`.
|
||||
|
||||
## Training an adapter on a collection
|
||||
|
||||
An adapter (LoRA) changes how a model *writes*: the organisation's words,
|
||||
format and tone. It does not reliably teach facts. On the PoC, a small
|
||||
model trained on nine policy documents answered 4 of 9 rephrased questions
|
||||
right and mixed facts up on the rest, while the same questions through the
|
||||
knowledge base got 9 of 9. Use both: the knowledge base for what answers
|
||||
say, an adapter for how they sound.
|
||||
|
||||
```sh
|
||||
# 1. Questions and answers from the collection, written by a model of the
|
||||
# cluster. Read the file through and correct it.
|
||||
llmgw ctl kb dataset Acme/policies --model qwen2.5-coder-14b-instruct --out policies.jsonl
|
||||
|
||||
# 2. Train on a node. --gpu-memory is what the job needs of a card; it
|
||||
# runs beside an engine when the engine leaves that much free.
|
||||
llmgw ctl train start policies-tone --model qwen2.5-0.5b-instruct \
|
||||
--base-repo Qwen/Qwen2.5-0.5B-Instruct --node cuda01 --gpu-memory 2300 --data policies.jsonl
|
||||
llmgw ctl train show JOB --follow
|
||||
|
||||
# 3. Make it the model's adapter, kept on the mirror.
|
||||
llmgw ctl train publish JOB --restart
|
||||
```
|
||||
|
||||
The node trains with its installed vLLM's torch and a small overlay (peft
|
||||
and accelerate), as the engines' user. For a model llama.cpp serves, it also
|
||||
converts the adapter to GGUF with the installed llama.cpp's own code. What
|
||||
it makes is kept by the control plane (`llmgw ctl train download`).
|
||||
|
||||
An adapter is private like its collection:
|
||||
|
||||
- **It has an owner.** An adapter trained on a collection carries some of
|
||||
what the collection says in its weights, so it belongs to a part of the
|
||||
tree.
|
||||
- By default the owner is the collection's owner.
|
||||
- `--owner` names another, which must be able to read every collection
|
||||
the pairs came from.
|
||||
- Only callers in the owner's part of the tree see the adapter. To anyone
|
||||
else it is a model that does not exist, absent from `/v1/models` too.
|
||||
- **It is public only by choice.** `--public` makes an adapter every
|
||||
caller's. It is refused for pairs from a knowledge base, so it is only for
|
||||
pairs written by hand.
|
||||
|
||||
## How it is kept apart
|
||||
|
||||
Passages live in PostgreSQL, partitioned by organization, with an index per
|
||||
collection. Every search and every write names the organization and the
|
||||
collections the caller may use. On top of that, row-level security on the
|
||||
table checks the same thing again, so a query that forgot its filter
|
||||
returns nothing rather than another tenant's passages.
|
||||
+27
-5
@@ -379,16 +379,38 @@ is what someone gets from a borrowed laptop, and the password is what they do no
|
||||
have — and it signs every other browser out, which is the point if they are
|
||||
changing it because something went wrong.
|
||||
|
||||
There is no self-service reset, because this gateway sends no email and a reset
|
||||
link with nowhere to send it is not a reset. When someone loses theirs:
|
||||
Someone who has forgotten theirs follows **Forgotten your password?** on the
|
||||
sign-in page. With mail set up in the cluster's settings, a link to set a new one
|
||||
is mailed to their address — the page says the same whether or not the address
|
||||
has an account, so nobody can find out who does from it:
|
||||
|
||||
```
|
||||
llmgw ctl config set mail '{smtp: smtp.example.com:587, from: llmgw@example.com,
|
||||
username: llmgw, password_file: /etc/llmgw/secrets/smtp, base_url: https://llm.example.com}'
|
||||
```
|
||||
|
||||
`base_url` is where the link points, and it is never taken from the request:
|
||||
otherwise a stranger could ask for someone's link from a page of their own. A
|
||||
link works once, for an hour, and a newer one replaces it.
|
||||
|
||||
Without mail, the page says to ask an operator, who makes the link and hands it
|
||||
over however suits:
|
||||
|
||||
```
|
||||
llmgw users reset-link ayse@example.com --store … # --base-url without mail.base_url
|
||||
```
|
||||
|
||||
Or sets a temporary password to give them, and has them change it from their
|
||||
profile:
|
||||
|
||||
```
|
||||
llmgw users reset-password usr-k4mq… --password "a temporary passphrase" --store …
|
||||
```
|
||||
|
||||
Tell them out of band and have them change it from their profile. Their sessions
|
||||
are closed — if they lost the password, the reason might be that someone else has
|
||||
it. Their API keys keep working; those are separate credentials.
|
||||
Either way their sessions are closed — if they lost the password, the reason
|
||||
might be that someone else has it. Their API keys keep working; those are
|
||||
separate credentials. People who sign in through single sign-on have no password
|
||||
here, and get no link.
|
||||
|
||||
## Trying a model in the browser
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ install:
|
||||
- your administrator account. **The password is shown once. Copy it now.**
|
||||
- a model mirror, which downloads model files once for the whole cluster;
|
||||
- this machine as a first node, and a small model (Qwen2.5 0.5B) on it;
|
||||
- guardrails on every request, with a judge model (Llama Guard 3 1B, two
|
||||
- guardrails on every request, with a judge model (Qwen3Guard-Gen 0.6B, two
|
||||
replicas) on CPUs — a request it cannot check is refused, not let through — and API keys that expire within 90 days. What is hardened, and how
|
||||
to loosen it: [install.md](install.md#2-the-control-plane).
|
||||
|
||||
|
||||
+666
-5
@@ -18,6 +18,7 @@ rather than repeated.
|
||||
| `name` | `string` | [Adapter](#adapter) |
|
||||
| `source` | `string` | [Adapter](#adapter) |
|
||||
| `scale` | `float64` | [Adapter](#adapter) |
|
||||
| `owner` | `string` | [Adapter](#adapter) |
|
||||
| `enabled` | `bool` | [Admission](#admission) |
|
||||
| `bands` | `[]Band` | [Admission](#admission) |
|
||||
| `default_band` | `string` | [Admission](#admission) |
|
||||
@@ -26,6 +27,8 @@ rather than repeated.
|
||||
| `queue_capacity_per_band` | `int` | [Admission](#admission) |
|
||||
| `queue_capacity_total` | `int` | [Admission](#admission) |
|
||||
| `interval` | `time.Duration` | [Admission](#admission) |
|
||||
| `default` | `map[string]any` | [AgentSettings](#agentsettings) |
|
||||
| `nodes` | `map[string]map[string]any` | [AgentSettings](#agentsettings) |
|
||||
| `enabled` | `bool` | [Auth](#auth) |
|
||||
| `store` | `string` | [Auth](#auth) |
|
||||
| `cache_ttl` | `time.Duration` | [Auth](#auth) |
|
||||
@@ -57,6 +60,17 @@ rather than repeated.
|
||||
| `tools` | `bool` | [Capabilities](#capabilities) |
|
||||
| `vision` | `bool` | [Capabilities](#capabilities) |
|
||||
| `embeddings` | `bool` | [Capabilities](#capabilities) |
|
||||
| `images` | `bool` | [Capabilities](#capabilities) |
|
||||
| `meshes` | `bool` | [Capabilities](#capabilities) |
|
||||
| `workflow` | `string` | [ComfyUI](#comfyui) |
|
||||
| `checkpoint` | `string` | [ComfyUI](#comfyui) |
|
||||
| `output` | `string` | [ComfyUI](#comfyui) |
|
||||
| `inputs` | `[]Input` | [ComfyUI](#comfyui) |
|
||||
| `nodes` | `[]CustomNode` | [ComfyUI](#comfyui) |
|
||||
| `huggingface_repo` | `string` | [ComfyUI](#comfyui) |
|
||||
| `environment` | `string` | [ComfyUI](#comfyui) |
|
||||
| `dir` | `string` | [ComfyUIConfig](#comfyuiconfig) |
|
||||
| `python` | `string` | [ComfyUIConfig](#comfyuiconfig) |
|
||||
| `interval` | `time.Duration` | [Control](#control) |
|
||||
| `base_backoff` | `time.Duration` | [Control](#control) |
|
||||
| `max_backoff` | `time.Duration` | [Control](#control) |
|
||||
@@ -68,11 +82,16 @@ rather than repeated.
|
||||
| `eval_dir` | `string` | [Control](#control) |
|
||||
| `enabled` | `bool` | [Control](#control) |
|
||||
| `lease_ttl` | `time.Duration` | [Control](#control) |
|
||||
| `repo` | `string` | [CustomNode](#customnode) |
|
||||
| `commit` | `string` | [CustomNode](#customnode) |
|
||||
| `requirements` | `[]string` | [CustomNode](#customnode) |
|
||||
| `build` | `[]string` | [CustomNode](#customnode) |
|
||||
| `id` | `string` | [Endpoint](#endpoint) |
|
||||
| `url` | `string` | [Endpoint](#endpoint) |
|
||||
| `api_key` | `string` | [Endpoint](#endpoint) |
|
||||
| `models` | `[]string` | [Endpoint](#endpoint) |
|
||||
| `engine` | `string` | [Endpoint](#endpoint) |
|
||||
| `workflow` | `string` | [Endpoint](#endpoint) |
|
||||
| `pool` | `string` | [Endpoint](#endpoint) |
|
||||
| `weight` | `float64` | [Endpoint](#endpoint) |
|
||||
| `priority` | `int` | [Endpoint](#endpoint) |
|
||||
@@ -87,6 +106,9 @@ rather than repeated.
|
||||
| `model` | `string` | [External](#external) |
|
||||
| `api_key_file` | `string` | [External](#external) |
|
||||
| `peer` | `string` | [External](#external) |
|
||||
| `api` | `string` | [External](#external) |
|
||||
| `workflow` | `string` | [External](#external) |
|
||||
| `output` | `string` | [External](#external) |
|
||||
| `budgets` | `[]FacilityBudget` | [Facility](#facility) |
|
||||
| `margin` | `float64` | [Facility](#facility) |
|
||||
| `gpu_watts` | `float64` | [Facility](#facility) |
|
||||
@@ -99,12 +121,25 @@ rather than repeated.
|
||||
| `name` | `string` | [Federation](#federation) |
|
||||
| `interval` | `time.Duration` | [Federation](#federation) |
|
||||
| `enabled` | `bool` | [FirewallSettings](#firewallsettings) |
|
||||
| `min_compute_capability` | `float64` | [GPUNeed](#gpuneed) |
|
||||
| `min_memory_gb` | `int` | [GPUNeed](#gpuneed) |
|
||||
| `format` | `string` | [GuardModel](#guardmodel) |
|
||||
| `categories` | `[]string` | [GuardModel](#guardmodel) |
|
||||
| `name` | `string` | [Input](#input) |
|
||||
| `type` | `string` | [Input](#input) |
|
||||
| `required` | `bool` | [Input](#input) |
|
||||
| `default` | `any` | [Input](#input) |
|
||||
| `min` | `*float64` | [Input](#input) |
|
||||
| `max` | `*float64` | [Input](#input) |
|
||||
| `choices` | `[]string` | [Input](#input) |
|
||||
| `max_mb` | `int` | [Input](#input) |
|
||||
| `doc` | `string` | [Input](#input) |
|
||||
| `model` | `string` | [JudgePolicy](#judgepolicy) |
|
||||
| `models` | `[]string` | [JudgePolicy](#judgepolicy) |
|
||||
| `input` | `string` | [JudgePolicy](#judgepolicy) |
|
||||
| `output` | `string` | [JudgePolicy](#judgepolicy) |
|
||||
| `categories` | `[]string` | [JudgePolicy](#judgepolicy) |
|
||||
| `controversial` | `string` | [JudgePolicy](#judgepolicy) |
|
||||
| `latency_budget` | `time.Duration` | [JudgePolicy](#judgepolicy) |
|
||||
| `tokenize_timeout` | `time.Duration` | [KVIndex](#kvindex) |
|
||||
| `min_blocks` | `int` | [KVIndex](#kvindex) |
|
||||
@@ -113,6 +148,14 @@ rather than repeated.
|
||||
| `notice` | `string` | [Landing](#landing) |
|
||||
| `language` | `string` | [Landing](#landing) |
|
||||
| `docs_sign_in` | `bool` | [Landing](#landing) |
|
||||
| `id` | `string` | [License](#license) |
|
||||
| `name` | `string` | [License](#license) |
|
||||
| `url` | `string` | [License](#license) |
|
||||
| `notice` | `string` | [License](#license) |
|
||||
| `commercial` | `*bool` | [License](#license) |
|
||||
| `excluded_regions` | `[]string` | [License](#license) |
|
||||
| `revenue_cap_usd` | `int64` | [License](#license) |
|
||||
| `attribution` | `string` | [License](#license) |
|
||||
| `bin` | `string` | [LlamaCppConfig](#llamacppconfig) |
|
||||
| `parallel` | `int` | [LlamaCppConfig](#llamacppconfig) |
|
||||
| `threads` | `int` | [LlamaCppConfig](#llamacppconfig) |
|
||||
@@ -130,6 +173,8 @@ rather than repeated.
|
||||
| `class` | `string` | [Model](#model) |
|
||||
| `owner` | `string` | [Model](#model) |
|
||||
| `external` | `*External` | [Model](#model) |
|
||||
| `comfyui` | `*ComfyUI` | [Model](#model) |
|
||||
| `license` | `*License` | [Model](#model) |
|
||||
| `adapters` | `[]Adapter` | [Model](#model) |
|
||||
| `max_loaded_adapters` | `int` | [Model](#model) |
|
||||
| `fallbacks` | `[]string` | [Model](#model) |
|
||||
@@ -140,6 +185,7 @@ rather than repeated.
|
||||
| `context_length` | `int` | [Model](#model) |
|
||||
| `max_output` | `int` | [Model](#model) |
|
||||
| `capabilities` | `Capabilities` | [Model](#model) |
|
||||
| `query_prefix` | `string` | [Model](#model) |
|
||||
| `role` | `string` | [Model](#model) |
|
||||
| `guard` | `*GuardModel` | [Model](#model) |
|
||||
| `origin` | `*Origin` | [Model](#model) |
|
||||
@@ -159,6 +205,8 @@ rather than repeated.
|
||||
| `no_passwords` | `bool` | [OIDC](#oidc) |
|
||||
| `mirror` | `string` | [Offline](#offline) |
|
||||
| `checksums` | `string` | [Offline](#offline) |
|
||||
| `evict_after_idle` | `time.Duration` | [OnDemandSettings](#ondemandsettings) |
|
||||
| `min_lifetime` | `time.Duration` | [OnDemandSettings](#ondemandsettings) |
|
||||
| `nvidia_smi` | `string` | [Options](#options) |
|
||||
| `ib_root` | `string` | [Options](#options) |
|
||||
| `prefer_nvml` | `bool` | [Options](#options) |
|
||||
@@ -198,12 +246,15 @@ rather than repeated.
|
||||
| `match_ratio` | `float64` | [Prefix](#prefix) |
|
||||
| `balance_abs` | `float64` | [Prefix](#prefix) |
|
||||
| `balance_rel` | `float64` | [Prefix](#prefix) |
|
||||
| `auto` | `bool` | [PresetSettings](#presetsettings) |
|
||||
| `source` | `string` | [PreviousOrigin](#previousorigin) |
|
||||
| `revision` | `string` | [PreviousOrigin](#previousorigin) |
|
||||
| `files` | `[]OriginFile` | [PreviousOrigin](#previousorigin) |
|
||||
| `name` | `string` | [Profile](#profile) |
|
||||
| `engine` | `string` | [Profile](#profile) |
|
||||
| `engine_version` | `string` | [Profile](#profile) |
|
||||
| `gpu_class` | `string` | [Profile](#profile) |
|
||||
| `source` | `string` | [Profile](#profile) |
|
||||
| `tp` | `int` | [Profile](#profile) |
|
||||
| `pp` | `int` | [Profile](#profile) |
|
||||
| `gpus_per_node` | `int` | [Profile](#profile) |
|
||||
@@ -220,6 +271,11 @@ rather than repeated.
|
||||
| `priority` | `int` | [Profile](#profile) |
|
||||
| `startup_timeout` | `time.Duration` | [Profile](#profile) |
|
||||
| `kv_events` | `bool` | [Profile](#profile) |
|
||||
| `gpu` | `*GPUNeed` | [Profile](#profile) |
|
||||
| `comfyui` | `*ProfileComfyUI` | [Profile](#profile) |
|
||||
| `workflow` | `string` | [ProfileComfyUI](#profilecomfyui) |
|
||||
| `checkpoint` | `string` | [ProfileComfyUI](#profilecomfyui) |
|
||||
| `nodes` | `[]CustomNode` | [ProfileComfyUI](#profilecomfyui) |
|
||||
| `activation_wait` | `time.Duration` | [Router](#router) |
|
||||
| `scrape_interval` | `time.Duration` | [Router](#router) |
|
||||
| `scrape_timeout` | `time.Duration` | [Router](#router) |
|
||||
@@ -288,7 +344,12 @@ rather than repeated.
|
||||
| `guard` | `guard.Config` | [Server](#server) |
|
||||
| `secrets` | `Secrets` | [Server](#server) |
|
||||
| `notify` | `notify.Config` | [Server](#server) |
|
||||
| `mail` | `mail.Config` | [Server](#server) |
|
||||
| `serving` | `ServingSettings` | [Server](#server) |
|
||||
| `firewall` | `FirewallSettings` | [Server](#server) |
|
||||
| `presets` | `PresetSettings` | [Server](#server) |
|
||||
| `on_demand` | `OnDemandSettings` | [Server](#server) |
|
||||
| `agents` | `AgentSettings` | [Server](#server) |
|
||||
| `firewall_networks` | `[]string` | [Server](#server) |
|
||||
| `backup_recipients` | `[]string` | [Server](#server) |
|
||||
| `backup_copy_to` | `[]string` | [Server](#server) |
|
||||
@@ -300,6 +361,7 @@ rather than repeated.
|
||||
| `facility` | `Facility` | [Server](#server) |
|
||||
| `federation` | `Federation` | [Server](#server) |
|
||||
| `endpoints` | `[]Endpoint` | [Server](#server) |
|
||||
| `stopped` | `bool` | [ServingSettings](#servingsettings) |
|
||||
| `bin` | `string` | [SimConfig](#simconfig) |
|
||||
| `python` | `string` | [TRTLLMConfig](#trtllmconfig) |
|
||||
| `store` | `string` | [Threads](#threads) |
|
||||
@@ -332,9 +394,12 @@ rather than repeated.
|
||||
| `secret_file` | `string` | [Webhook](#webhook) |
|
||||
| `prompt_per_1k` | `int64` | [accounting.Price](#accountingprice) |
|
||||
| `completion_per_1k` | `int64` | [accounting.Price](#accountingprice) |
|
||||
| `per_image` | `int64` | [accounting.Price](#accountingprice) |
|
||||
| `per_mesh` | `int64` | [accounting.Price](#accountingprice) |
|
||||
| `name` | `string` | [admission.Band](#admissionband) |
|
||||
| `priority` | `int` | [admission.Band](#admissionband) |
|
||||
| `saturation_ceiling` | `float64` | [admission.Band](#admissionband) |
|
||||
| `queue_ttl` | `time.Duration` | [admission.Band](#admissionband) |
|
||||
| `bands` | `[]Band` | [admission.Config](#admissionconfig) |
|
||||
| `default_band` | `string` | [admission.Config](#admissionconfig) |
|
||||
| `queue_ttl` | `time.Duration` | [admission.Config](#admissionconfig) |
|
||||
@@ -364,17 +429,26 @@ rather than repeated.
|
||||
| `name` | `string` | [config.Band](#configband) |
|
||||
| `priority` | `int` | [config.Band](#configband) |
|
||||
| `saturation_ceiling` | `float64` | [config.Band](#configband) |
|
||||
| `queue_ttl` | `time.Duration` | [config.Band](#configband) |
|
||||
| `prompt_per_1k` | `int64` | [config.Price](#configprice) |
|
||||
| `completion_per_1k` | `int64` | [config.Price](#configprice) |
|
||||
| `per_image` | `int64` | [config.Price](#configprice) |
|
||||
| `per_mesh` | `int64` | [config.Price](#configprice) |
|
||||
| `disabled` | `[]string` | [engines.Config](#enginesconfig) |
|
||||
| `vllm` | `VLLMConfig` | [engines.Config](#enginesconfig) |
|
||||
| `sglang` | `SGLangConfig` | [engines.Config](#enginesconfig) |
|
||||
| `llamacpp` | `LlamaCppConfig` | [engines.Config](#enginesconfig) |
|
||||
| `trtllm` | `TRTLLMConfig` | [engines.Config](#enginesconfig) |
|
||||
| `sim` | `SimConfig` | [engines.Config](#enginesconfig) |
|
||||
| `comfyui` | `ComfyUIConfig` | [engines.Config](#enginesconfig) |
|
||||
| `enabled` | `*bool` | [guard.Config](#guardconfig) |
|
||||
| `default` | `[]string` | [guard.Config](#guardconfig) |
|
||||
| `policies` | `map[string]Policy` | [guard.Config](#guardconfig) |
|
||||
| `smtp` | `string` | [mail.Config](#mailconfig) |
|
||||
| `from` | `string` | [mail.Config](#mailconfig) |
|
||||
| `username` | `string` | [mail.Config](#mailconfig) |
|
||||
| `password_file` | `string` | [mail.Config](#mailconfig) |
|
||||
| `base_url` | `string` | [mail.Config](#mailconfig) |
|
||||
| `webhooks` | `[]Webhook` | [notify.Config](#notifyconfig) |
|
||||
| `host` | `string` | [notify.Config](#notifyconfig) |
|
||||
| `work_dir` | `string` | [supervisor.Config](#supervisorconfig) |
|
||||
@@ -412,6 +486,16 @@ weights are.
|
||||
|
||||
Scale is how strongly it is applied; zero takes 1.
|
||||
|
||||
### `owner`
|
||||
|
||||
`string`
|
||||
|
||||
Owner is the part of the tenancy tree the adapter belongs to —
|
||||
org:ID, team:ID, project:ID, user:ID or key:ID — and only callers in
|
||||
it see it: to anyone else it is a model that does not exist. An
|
||||
adapter trained on an organisation's documents carries some of what
|
||||
they say in its weights. Empty is every caller the model allows.
|
||||
|
||||
## Admission
|
||||
|
||||
### `enabled`
|
||||
@@ -458,6 +542,21 @@ Zero takes half of all the bands' capacities together.
|
||||
|
||||
`time.Duration`
|
||||
|
||||
## AgentSettings
|
||||
|
||||
### `default`
|
||||
|
||||
`map[string]any`
|
||||
|
||||
Default is agent.yaml's keys for every node.
|
||||
|
||||
### `nodes`
|
||||
|
||||
`map[string]map[string]any`
|
||||
|
||||
Nodes are agent.yaml's keys for one node each, by node id, over
|
||||
Default.
|
||||
|
||||
## Auth
|
||||
|
||||
### `enabled`
|
||||
@@ -675,6 +774,97 @@ token is MaxTTFTRatio times this model's. Zero is 50, 0.05 and 1.5.
|
||||
|
||||
`bool`
|
||||
|
||||
### `images`
|
||||
|
||||
`bool`
|
||||
|
||||
Images is a model that makes pictures, asked for at
|
||||
/v1/images/generations. Implied by an external model whose API is
|
||||
ComfyUI's.
|
||||
|
||||
### `meshes`
|
||||
|
||||
`bool`
|
||||
|
||||
Meshes is a model that makes 3D models from a picture, asked for as a
|
||||
job at /v1/3d/generations. Implied by a ComfyUI model whose output is
|
||||
meshes.
|
||||
|
||||
## ComfyUI
|
||||
|
||||
### `workflow`
|
||||
|
||||
`string`
|
||||
|
||||
Workflow is a built-in's name (flux-schnell, the default, or
|
||||
flux-dev) or a JSON file on the control plane, in ComfyUI's API
|
||||
format.
|
||||
|
||||
### `checkpoint`
|
||||
|
||||
`string`
|
||||
|
||||
Checkpoint is the file the workflow's {{checkpoint}} names: the
|
||||
source's own name unless set, which a source that is a set of files
|
||||
must.
|
||||
|
||||
### `output`
|
||||
|
||||
`string`
|
||||
|
||||
Output is what the workflow makes: pictures (the default), asked for
|
||||
at /v1/images/generations; meshes, asked for as a job at
|
||||
/v1/3d/generations from a picture; or video, audio or files. Every
|
||||
kind is asked for as a job at /v1/workflows/runs too.
|
||||
|
||||
### `inputs`
|
||||
|
||||
`[]Input`
|
||||
|
||||
Inputs are what the workflow takes from a request at
|
||||
/v1/workflows/runs, each the placeholder of its name.
|
||||
|
||||
### `nodes`
|
||||
|
||||
`[]CustomNode`
|
||||
|
||||
Nodes are custom nodes the workflow needs, put in each replica's own
|
||||
custom_nodes at the commit named.
|
||||
|
||||
### `huggingface_repo`
|
||||
|
||||
`string`
|
||||
|
||||
HuggingFaceRepo is the repository a custom node loads the weights by
|
||||
name from (Stable Fast 3D's loader asks for stabilityai/stable-fast-3d):
|
||||
the replica's weights are shown to the node there, in the Hugging Face
|
||||
cache of the engines' user, so nothing is fetched and no token is
|
||||
needed when it runs.
|
||||
|
||||
### `environment`
|
||||
|
||||
`string`
|
||||
|
||||
Environment names the models that may share one ComfyUI replica: every
|
||||
model of an environment agrees on the custom nodes they both name, so
|
||||
one replica can carry all of them and ComfyUI moves between their
|
||||
weights itself. Empty is an environment of the model alone.
|
||||
|
||||
## ComfyUIConfig
|
||||
|
||||
### `dir`
|
||||
|
||||
`string`
|
||||
|
||||
Dir is ComfyUI's directory, where main.py is. Readable by the user
|
||||
engines run as.
|
||||
|
||||
### `python`
|
||||
|
||||
`string`
|
||||
|
||||
Python is the interpreter of its environment; Dir's .venv when unset.
|
||||
|
||||
## Control
|
||||
|
||||
### `interval`
|
||||
@@ -760,6 +950,42 @@ see the claim. Long enough to survive a slow query or a brief network
|
||||
stall, short enough that a standby takes over before anyone has finished
|
||||
reading the alert. Zero takes the default.
|
||||
|
||||
## CustomNode
|
||||
|
||||
### `repo`
|
||||
|
||||
`string`
|
||||
|
||||
Repo is the repository, https only.
|
||||
|
||||
### `commit`
|
||||
|
||||
`string`
|
||||
|
||||
Commit is the full commit the node is used at: a branch moves, and a
|
||||
node that changed under a model is a model that changed unasked.
|
||||
|
||||
### `requirements`
|
||||
|
||||
`[]string`
|
||||
|
||||
Requirements are the PyPI packages the node needs, by name and version
|
||||
range only. They are installed beside the replica, never into
|
||||
ComfyUI's environment: a package named here at the range named, and
|
||||
every other one at the version ComfyUI already runs. One named here
|
||||
is first on ComfyUI's path, so its range must suit ComfyUI too.
|
||||
|
||||
### `build`
|
||||
|
||||
`[]string`
|
||||
|
||||
Build are directories of the repository that are Python packages to
|
||||
be compiled rather than downloaded: C++ and CUDA extensions no index
|
||||
has (Stable Fast 3D's texture_baker and uv_unwrapper). Each is built
|
||||
as a wheel with ComfyUI's Python for the card the replica runs on,
|
||||
kept on the node for the next replica on the same kind of card, and
|
||||
installed beside the replica like its requirements.
|
||||
|
||||
## Endpoint
|
||||
|
||||
### `id`
|
||||
@@ -788,6 +1014,13 @@ with one.
|
||||
Engine names the inference engine, which decides what its metrics are
|
||||
called. Empty means vLLM's names.
|
||||
|
||||
### `workflow`
|
||||
|
||||
`string`
|
||||
|
||||
Workflow is the ComfyUI workflow for engine: comfyui, a built-in's
|
||||
name or its JSON; flux-schnell unless set.
|
||||
|
||||
### `pool`
|
||||
|
||||
`string`
|
||||
@@ -879,6 +1112,29 @@ a model that federation added (`llmgw ctl federation add`) rather
|
||||
than one written by hand. Such a model is named PEER/MODEL, comes and
|
||||
goes with the peer's own list, and is sent only while the peer is up.
|
||||
|
||||
### `api`
|
||||
|
||||
`string`
|
||||
|
||||
API is what the provider speaks: OpenAI's, unless this says comfyui —
|
||||
a ComfyUI server, asked for pictures at /v1/images/generations and
|
||||
sent the workflow below with the request's prompt in it. Model is then
|
||||
the checkpoint file the workflow loads, as ComfyUI lists it.
|
||||
|
||||
### `workflow`
|
||||
|
||||
`string`
|
||||
|
||||
Workflow is the ComfyUI workflow, in its API format: a built-in's name
|
||||
(flux-schnell, the default, or flux-dev) or a JSON file on the control
|
||||
plane.
|
||||
|
||||
### `output`
|
||||
|
||||
`string`
|
||||
|
||||
Output is what it makes: pictures, or meshes (see ComfyUI.Output).
|
||||
|
||||
## Facility
|
||||
|
||||
### `budgets`
|
||||
@@ -979,15 +1235,32 @@ site whose own firewall does that job; the rules are kept, and true
|
||||
puts them back. Changed in the cluster's settings, it is in force on
|
||||
every node and control plane within seconds.
|
||||
|
||||
## GPUNeed
|
||||
|
||||
### `min_compute_capability`
|
||||
|
||||
`float64`
|
||||
|
||||
MinComputeCapability is the oldest architecture that runs it: 7.5 is
|
||||
Turing, 8.0 Ampere (bf16, flash-attention 2), 8.9 Ada (fp8), 9.0 Hopper.
|
||||
|
||||
### `min_memory_gb`
|
||||
|
||||
`int`
|
||||
|
||||
MinMemoryGB is the card's own memory it needs.
|
||||
|
||||
## GuardModel
|
||||
|
||||
### `format`
|
||||
|
||||
`string`
|
||||
|
||||
Format is llamaguard3 (Meta's Llama Guard 3: "safe", or "unsafe" and
|
||||
the categories), shieldgemma (Google's ShieldGemma: Yes or No), or
|
||||
json ({"safe": false, "categories": ["..."]}).
|
||||
Format is qwen3guard (Qwen3Guard-Gen: "Safety: Safe", "Unsafe" or
|
||||
"Controversial", and the categories), llamaguard3 (Meta's Llama Guard
|
||||
3: "safe", or "unsafe" and the categories), shieldgemma (Google's
|
||||
ShieldGemma: Yes or No), or json ({"safe": false, "categories":
|
||||
["..."]}).
|
||||
|
||||
### `categories`
|
||||
|
||||
@@ -996,6 +1269,62 @@ json ({"safe": false, "categories": ["..."]}).
|
||||
Categories the model is asked about, for shieldgemma — each a policy
|
||||
sentence — and the names reported for llamaguard3's S1, S2, ….
|
||||
|
||||
## Input
|
||||
|
||||
### `name`
|
||||
|
||||
`string`
|
||||
|
||||
Name is the placeholder's, and the request's key.
|
||||
|
||||
### `type`
|
||||
|
||||
`string`
|
||||
|
||||
Type is one of text, int, float, bool, choice, image, mask, audio,
|
||||
video, file. The last five are files, sent as base64, a data URL, or
|
||||
the id of a file uploaded to /v1/files, and put in ComfyUI's input
|
||||
directory; the placeholder gets the name they were given there.
|
||||
|
||||
### `required`
|
||||
|
||||
`bool`
|
||||
|
||||
Required inputs must be in every request; the others take Default,
|
||||
or the workflow's own {{name|default}}.
|
||||
|
||||
### `default`
|
||||
|
||||
`any`
|
||||
|
||||
### `min`
|
||||
|
||||
`*float64`
|
||||
|
||||
Min and Max bound a number; for text, Max bounds its length.
|
||||
|
||||
### `max`
|
||||
|
||||
`*float64`
|
||||
|
||||
### `choices`
|
||||
|
||||
`[]string`
|
||||
|
||||
Choices are what a choice may be.
|
||||
|
||||
### `max_mb`
|
||||
|
||||
`int`
|
||||
|
||||
MaxMB bounds a file input; zero is the type's own bound.
|
||||
|
||||
### `doc`
|
||||
|
||||
`string`
|
||||
|
||||
Doc says what the input is for, to whoever fills it in.
|
||||
|
||||
## JudgePolicy
|
||||
|
||||
### `model`
|
||||
@@ -1004,6 +1333,15 @@ sentence — and the names reported for llamaguard3's S1, S2, ….
|
||||
|
||||
Model is a catalog model with role: guard.
|
||||
|
||||
### `models`
|
||||
|
||||
`[]string`
|
||||
|
||||
Models are judges asked in turn, instead of Model: the first that
|
||||
answers gives the verdict. A judge on a GPU first and one on CPUs
|
||||
after it is the GPU's speed while it is there, and the CPUs' when it
|
||||
is not — busy, failed, or the node down.
|
||||
|
||||
### `input`
|
||||
|
||||
`string`
|
||||
@@ -1022,12 +1360,19 @@ Empty does not ask the judge about that direction.
|
||||
Categories, when set, are the only ones that count: Llama Guard's
|
||||
codes (S1, S10) or the names the model reports.
|
||||
|
||||
### `controversial`
|
||||
|
||||
`string`
|
||||
|
||||
Controversial is what Qwen3Guard's third verdict counts as: safe
|
||||
(the default) or unsafe.
|
||||
|
||||
### `latency_budget`
|
||||
|
||||
`time.Duration`
|
||||
|
||||
LatencyBudget is how long the judge is waited for before the policy's
|
||||
fail applies. Zero is 250ms.
|
||||
LatencyBudget is how long each judge is waited for before the next is
|
||||
asked, or, after the last, the policy's fail applies. Zero is 250ms.
|
||||
|
||||
## KVIndex
|
||||
|
||||
@@ -1091,6 +1436,63 @@ llmgw in general — no address, model or key of this installation —
|
||||
which the project's repository publishes anyway. On, a visitor is sent
|
||||
to sign in, and without accounts the portal needs an API key.
|
||||
|
||||
## License
|
||||
|
||||
### `id`
|
||||
|
||||
`string`
|
||||
|
||||
ID names the licence for a machine: an SPDX identifier where there is
|
||||
one (MIT, Apache-2.0, CC-BY-NC-4.0), else a lowercase name of the
|
||||
licence's own (stability-community, tencent-hunyuan-community). What a
|
||||
person's or a site's choice of licences matches on. Name is for people.
|
||||
|
||||
### `name`
|
||||
|
||||
`string`
|
||||
|
||||
Name is the licence's own: MIT, Apache-2.0.
|
||||
|
||||
### `url`
|
||||
|
||||
`string`
|
||||
|
||||
### `notice`
|
||||
|
||||
`string`
|
||||
|
||||
Notice is what must accompany what was made: attribution, terms on
|
||||
the output, what the weights were trained on.
|
||||
|
||||
### `commercial`
|
||||
|
||||
`*bool`
|
||||
|
||||
Commercial says whether what is made may be used commercially: nil
|
||||
is not said, which is not the same as yes.
|
||||
|
||||
### `excluded_regions`
|
||||
|
||||
`[]string`
|
||||
|
||||
ExcludedRegions are where the licence grants nothing: ISO 3166 codes,
|
||||
and EU for the union (Hunyuan's: EU, UK, KR).
|
||||
|
||||
### `revenue_cap_usd`
|
||||
|
||||
`int64`
|
||||
|
||||
RevenueCapUSD is the yearly revenue above which the licence asks for
|
||||
another agreement (Stability's community licence: 1000000). Zero is
|
||||
none.
|
||||
|
||||
### `attribution`
|
||||
|
||||
`string`
|
||||
|
||||
Attribution is what a product using the output must show: "Powered by
|
||||
Stability AI".
|
||||
|
||||
## LlamaCppConfig
|
||||
|
||||
### `bin`
|
||||
@@ -1247,6 +1649,25 @@ allowlist.
|
||||
|
||||
Takes a [External](#external) block.
|
||||
|
||||
### `comfyui`
|
||||
|
||||
`*ComfyUI`
|
||||
|
||||
ComfyUI is how a model whose profiles run engine comfyui makes its
|
||||
pictures: the workflow, and the file it loads.
|
||||
|
||||
Takes a [ComfyUI](#comfyui) block.
|
||||
|
||||
### `license`
|
||||
|
||||
`*License`
|
||||
|
||||
License is the terms of what the model makes, for a model whose
|
||||
output somebody ships: shown with every result and written into a
|
||||
mesh.
|
||||
|
||||
Takes a [License](#license) block.
|
||||
|
||||
### `adapters`
|
||||
|
||||
`[]Adapter`
|
||||
@@ -1331,6 +1752,16 @@ Capabilities is what this model can do beyond plain completion.
|
||||
|
||||
Takes a [Capabilities](#capabilities) block.
|
||||
|
||||
### `query_prefix`
|
||||
|
||||
`string`
|
||||
|
||||
QueryPrefix is put before a question an embedding model embeds for a
|
||||
knowledge-base search; documents are embedded without it. Models
|
||||
trained with an instruction on the query side want it, Qwen3-Embedding
|
||||
"Instruct: Given a web search query, retrieve relevant passages that
|
||||
answer the query\nQuery: " for one.
|
||||
|
||||
### `role`
|
||||
|
||||
`string`
|
||||
@@ -1484,6 +1915,25 @@ kept by llmgw bundle import and llmgw bundle trust; every engine file
|
||||
a node fetches from the mirror is checked against it. Read again
|
||||
whenever it changes. Empty is /etc/llmgw/offline.sha256.
|
||||
|
||||
## OnDemandSettings
|
||||
|
||||
### `evict_after_idle`
|
||||
|
||||
`time.Duration`
|
||||
|
||||
EvictAfterIdle is how long an on-demand replica must go unasked-for
|
||||
before a model that wants its card may have it. Zero keeps 2m.
|
||||
|
||||
### `min_lifetime`
|
||||
|
||||
`time.Duration`
|
||||
|
||||
MinLifetime is how long a replica that has just come up is safe from
|
||||
that, so two models do not take turns paying for loads. Zero keeps
|
||||
5m. On one card shared by a picture model and a 3D model, 1m each
|
||||
brings the wait between a picture and its mesh to about a minute.
|
||||
Both are in force on the leader within seconds.
|
||||
|
||||
## Options
|
||||
|
||||
### `nvidia_smi`
|
||||
@@ -1732,6 +2182,20 @@ before affinity is abandoned.
|
||||
|
||||
`float64`
|
||||
|
||||
## PresetSettings
|
||||
|
||||
### `auto`
|
||||
|
||||
`bool`
|
||||
|
||||
Auto has the leading control plane add a picture model (flux-schnell)
|
||||
and a 3D model (triposr) by itself, on demand, once a node has a card
|
||||
that fits one and the catalog has no model that makes the same kind of
|
||||
thing — so a new cluster's portal has its pictures and 3D pages
|
||||
without anybody writing a definition. Only where the cluster already
|
||||
installs ComfyUI (engines.comfyui). A preset somebody removed is not
|
||||
added again by itself. In force on the leader within a minute.
|
||||
|
||||
## PreviousOrigin
|
||||
|
||||
### `source`
|
||||
@@ -1756,10 +2220,28 @@ before affinity is abandoned.
|
||||
|
||||
`string`
|
||||
|
||||
### `engine_version`
|
||||
|
||||
`string`
|
||||
|
||||
EngineVersion restricts the profile to nodes whose installed engine's
|
||||
version starts with it: "0.11" takes vLLM 0.11.0 and 0.11.1, not
|
||||
0.10.2. Empty takes whatever version the node has.
|
||||
|
||||
### `gpu_class`
|
||||
|
||||
`string`
|
||||
|
||||
### `source`
|
||||
|
||||
`string`
|
||||
|
||||
Source is the weights this profile loads, when not the model's: an
|
||||
embedding model served by vLLM from its repository on a card and by
|
||||
llama.cpp from a GGUF on CPUs is one model with two sources. Empty
|
||||
loads the model's. Updates and canaries move the model's source; a
|
||||
profile with its own keeps it.
|
||||
|
||||
### `tp`
|
||||
|
||||
`int`
|
||||
@@ -1907,6 +2389,39 @@ is started with sha256_cbor block hashing and a fixed PYTHONHASHSEED,
|
||||
which the router's hashing assumes; arguments overriding either leave
|
||||
the index empty, and the agent says so in its log.
|
||||
|
||||
### `gpu`
|
||||
|
||||
`*GPUNeed`
|
||||
|
||||
GPU says what card the profile needs, in place of naming its class:
|
||||
it stands for every class in the cluster that meets it. A catalog
|
||||
entry written once then places sensibly on any site's cards.
|
||||
|
||||
Takes a [GPUNeed](#gpuneed) block.
|
||||
|
||||
### `comfyui`
|
||||
|
||||
`*ProfileComfyUI`
|
||||
|
||||
ComfyUI is this profile's own workflow, checkpoint and nodes, over the
|
||||
model's: an fp8 workflow where the card has fp8, fp16 where it has not.
|
||||
|
||||
Takes a [ProfileComfyUI](#profilecomfyui) block.
|
||||
|
||||
## ProfileComfyUI
|
||||
|
||||
### `workflow`
|
||||
|
||||
`string`
|
||||
|
||||
### `checkpoint`
|
||||
|
||||
`string`
|
||||
|
||||
### `nodes`
|
||||
|
||||
`[]CustomNode`
|
||||
|
||||
## Router
|
||||
|
||||
### `activation_wait`
|
||||
@@ -2457,6 +2972,24 @@ Prometheus and Alertmanager in between. Empty sends nothing.
|
||||
|
||||
Takes a [notify.Config](#notifyconfig) block.
|
||||
|
||||
### `mail`
|
||||
|
||||
`mail.Config`
|
||||
|
||||
Mail is where the gateway's few emails go out from: a link to set a
|
||||
forgotten password. Unset, none is sent and people ask an operator.
|
||||
|
||||
Takes a [mail.Config](#mailconfig) block.
|
||||
|
||||
### `serving`
|
||||
|
||||
`ServingSettings`
|
||||
|
||||
Serving stops and starts the whole cluster's service (llmgw stop, llmgw
|
||||
start).
|
||||
|
||||
Takes a [ServingSettings](#servingsettings) block.
|
||||
|
||||
### `firewall`
|
||||
|
||||
`FirewallSettings`
|
||||
@@ -2465,6 +2998,35 @@ Firewall turns llmgw's firewall on every machine on or off.
|
||||
|
||||
Takes a [FirewallSettings](#firewallsettings) block.
|
||||
|
||||
### `presets`
|
||||
|
||||
`PresetSettings`
|
||||
|
||||
Presets is how the models that come with llmgw reach the catalog.
|
||||
|
||||
Takes a [PresetSettings](#presetsettings) block.
|
||||
|
||||
### `on_demand`
|
||||
|
||||
`OnDemandSettings`
|
||||
|
||||
OnDemand is how models that load when asked for take turns on the
|
||||
cards.
|
||||
|
||||
Takes a [OnDemandSettings](#ondemandsettings) block.
|
||||
|
||||
### `agents`
|
||||
|
||||
`AgentSettings`
|
||||
|
||||
Agents are the nodes' agent settings: agent.yaml's keys but those an
|
||||
agent needs to start, for every node (default) and for one
|
||||
(nodes.<id>). Sent to the agents as they change; an agent whose
|
||||
settings changed starts again on them, its engines serving
|
||||
throughout.
|
||||
|
||||
Takes a [AgentSettings](#agentsettings) block.
|
||||
|
||||
### `firewall_networks`
|
||||
|
||||
`[]string`
|
||||
@@ -2555,6 +3117,18 @@ Endpoints is the phase-1 static replica list. It is only consulted when
|
||||
no model declares a profile, which is what makes a router-only smoke test
|
||||
against sim-engines possible without a control plane.
|
||||
|
||||
## ServingSettings
|
||||
|
||||
### `stopped`
|
||||
|
||||
`bool`
|
||||
|
||||
Stopped takes every model down and answers every request for one with
|
||||
503 (system_stopped), leaving the control planes, agents and mirror
|
||||
running and every model's definition as it is: llmgw stop sets it,
|
||||
llmgw start clears it, and the models come back as their catalog
|
||||
entries ask. In force on every control plane within seconds.
|
||||
|
||||
## SimConfig
|
||||
|
||||
### `bin`
|
||||
@@ -2781,6 +3355,21 @@ is sha256=<hex HMAC of the body>.
|
||||
|
||||
`int64`
|
||||
|
||||
### `per_image`
|
||||
|
||||
`int64`
|
||||
|
||||
PerImage is what one picture costs, for a model that makes them: a
|
||||
picture's cost is the card's time, which its prompt's few tokens do
|
||||
not measure.
|
||||
|
||||
### `per_mesh`
|
||||
|
||||
`int64`
|
||||
|
||||
PerMesh is what one 3D model costs: minutes of a card, which no
|
||||
token count measures.
|
||||
|
||||
## admission.Band
|
||||
|
||||
### `name`
|
||||
@@ -2800,6 +3389,15 @@ Priority orders bands; higher is served first.
|
||||
SaturationCeiling is the pool saturation above which this band waits.
|
||||
1.0 means "never hold this band back".
|
||||
|
||||
### `queue_ttl`
|
||||
|
||||
`time.Duration`
|
||||
|
||||
QueueTTL, set, is how long this band's requests may wait instead of
|
||||
the controller's: a band for bulk work — a knowledge-base import, a
|
||||
backfill — whose requests had rather wait ten minutes behind people
|
||||
than be turned away after thirty seconds.
|
||||
|
||||
## admission.Config
|
||||
|
||||
### `bands`
|
||||
@@ -3013,6 +3611,14 @@ Takes a [Tracing](#tracing) block.
|
||||
|
||||
`float64`
|
||||
|
||||
### `queue_ttl`
|
||||
|
||||
`time.Duration`
|
||||
|
||||
QueueTTL is how long this band's requests may wait for room; the
|
||||
admission's queue_ttl unless set. Long for a band of bulk work, which
|
||||
had rather wait behind people than be turned away.
|
||||
|
||||
## config.Price
|
||||
|
||||
### `prompt_per_1k`
|
||||
@@ -3023,6 +3629,19 @@ Takes a [Tracing](#tracing) block.
|
||||
|
||||
`int64`
|
||||
|
||||
### `per_image`
|
||||
|
||||
`int64`
|
||||
|
||||
PerImage is the cost of one picture, for a model that makes them
|
||||
(/v1/images/generations), on top of its prompt's tokens.
|
||||
|
||||
### `per_mesh`
|
||||
|
||||
`int64`
|
||||
|
||||
PerMesh is the cost of one 3D model (/v1/3d/generations).
|
||||
|
||||
## engines.Config
|
||||
|
||||
### `disabled`
|
||||
@@ -3066,6 +3685,14 @@ Takes a [TRTLLMConfig](#trtllmconfig) block.
|
||||
|
||||
Takes a [SimConfig](#simconfig) block.
|
||||
|
||||
### `comfyui`
|
||||
|
||||
`ComfyUIConfig`
|
||||
|
||||
ComfyUI makes pictures; offered by a node whose ComfyUI is named here.
|
||||
|
||||
Takes a [ComfyUIConfig](#comfyuiconfig) block.
|
||||
|
||||
## guard.Config
|
||||
|
||||
### `enabled`
|
||||
@@ -3083,6 +3710,40 @@ whatever is defined or attached. Unset is on.
|
||||
|
||||
`map[string]Policy`
|
||||
|
||||
## mail.Config
|
||||
|
||||
### `smtp`
|
||||
|
||||
`string`
|
||||
|
||||
SMTP is the relay, host:port: 587 with STARTTLS, 465 with TLS from the
|
||||
start, 25 as the relay allows. Empty sends nothing.
|
||||
|
||||
### `from`
|
||||
|
||||
`string`
|
||||
|
||||
From is the sender, llmgw@example.com or "llmgw <llmgw@example.com>".
|
||||
|
||||
### `username`
|
||||
|
||||
`string`
|
||||
|
||||
Username and PasswordFile log in to the relay, when it wants that.
|
||||
The password is a secret reference: a file, env:NAME or vault:path.
|
||||
|
||||
### `password_file`
|
||||
|
||||
`string`
|
||||
|
||||
### `base_url`
|
||||
|
||||
`string`
|
||||
|
||||
BaseURL is the gateway's public address, which links in mail point
|
||||
at: https://llm.example.com. Never taken from a request, whose Host a
|
||||
sender chooses and could point a reset link at their own site.
|
||||
|
||||
## notify.Config
|
||||
|
||||
### `webhooks`
|
||||
|
||||
@@ -18,6 +18,7 @@ require (
|
||||
github.com/knadh/koanf/providers/env/v2 v2.0.1
|
||||
github.com/knadh/koanf/providers/file v1.2.1
|
||||
github.com/knadh/koanf/v2 v2.3.6
|
||||
github.com/ledongthuc/pdf v0.0.0-20260907135840-6c8c28e0e8a0
|
||||
github.com/mattn/go-isatty v0.0.24
|
||||
github.com/prometheus/client_golang v1.24.1
|
||||
github.com/prometheus/client_model v0.6.3
|
||||
@@ -34,6 +35,7 @@ require (
|
||||
golang.org/x/net v0.59.0
|
||||
golang.org/x/oauth2 v0.37.0
|
||||
golang.org/x/sync v0.23.0
|
||||
golang.org/x/term v0.46.0
|
||||
golang.org/x/text v0.42.0
|
||||
google.golang.org/protobuf v1.36.12
|
||||
modernc.org/sqlite v1.59.0
|
||||
|
||||
@@ -95,6 +95,8 @@ github.com/knadh/koanf/v2 v2.3.6 h1:JoQPSJmvS4aP0xNc8xMDr5tcrkSEInL23/Il7pITAKo=
|
||||
github.com/knadh/koanf/v2 v2.3.6/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/ledongthuc/pdf v0.0.0-20260907135840-6c8c28e0e8a0 h1:7Q+xNAZFmnfYOMweHN3c/PDFUKKfY1pVJ26K++QvVfU=
|
||||
github.com/ledongthuc/pdf v0.0.0-20260907135840-6c8c28e0e8a0/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||
|
||||
@@ -448,3 +448,40 @@ func TestAHashThatCannotGetATurnSaysTheServerIsBusy(t *testing.T) {
|
||||
t.Errorf("a hash with no turn: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAForgottenPasswordIsResetOnceThroughALink(t *testing.T) {
|
||||
svc, st := newService(t)
|
||||
ctx := context.Background()
|
||||
must(t, svc.SignUp(ctx, "ayse@example.com", "A", goodPassword))
|
||||
u, _ := st.UserByEmail(ctx, "ayse@example.com")
|
||||
|
||||
// Waiting for approval: no link, said the same as for nobody.
|
||||
if _, _, err := svc.RequestReset(ctx, st, "ayse@example.com"); !errors.Is(err, ErrNoReset) {
|
||||
t.Fatalf("a pending account got a link: %v", err)
|
||||
}
|
||||
if _, _, err := svc.RequestReset(ctx, st, "nobody@example.com"); !errors.Is(err, ErrNoReset) {
|
||||
t.Fatalf("an unknown address: %v", err)
|
||||
}
|
||||
must(t, st.ApproveUser(ctx, u.ID, "proj", "ilker", time.Now()))
|
||||
session, _, err := svc.SignIn(ctx, "ayse@example.com", goodPassword)
|
||||
must(t, err)
|
||||
|
||||
token, got, err := svc.RequestReset(ctx, st, "AYSE@example.com")
|
||||
if err != nil || token == "" || got.ID != u.ID {
|
||||
t.Fatalf("an active account: %q %+v %v", token, got, err)
|
||||
}
|
||||
if err := svc.ResetPassword(ctx, st, st, token, "short"); !errors.Is(err, ErrWeakPassword) {
|
||||
t.Fatalf("a weak password: %v", err)
|
||||
}
|
||||
next := goodPassword + "-new"
|
||||
must(t, svc.ResetPassword(ctx, st, st, token, next))
|
||||
if _, _, err := svc.Resolve(ctx, session); err == nil {
|
||||
t.Error("a session from before the reset still works")
|
||||
}
|
||||
if _, _, err := svc.SignIn(ctx, "ayse@example.com", next); err != nil {
|
||||
t.Errorf("the new password does not sign in: %v", err)
|
||||
}
|
||||
if err := svc.ResetPassword(ctx, st, st, token, next+"2"); !errors.Is(err, ErrNoReset) {
|
||||
t.Errorf("a link used twice: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -373,3 +373,63 @@ func (s *Service) OpenSession(ctx context.Context, userID string) (token, csrf s
|
||||
}
|
||||
return token, csrf, nil
|
||||
}
|
||||
|
||||
// ResetLinkTTL is how long a password reset link works.
|
||||
const ResetLinkTTL = time.Hour
|
||||
|
||||
// ResetStore keeps password reset links.
|
||||
type ResetStore interface {
|
||||
UserByEmail(ctx context.Context, email string) (store.User, error)
|
||||
CreatePasswordReset(ctx context.Context, userID string, ttl time.Duration) (string, error)
|
||||
PasswordResetUser(ctx context.Context, token string) (string, error)
|
||||
UsePasswordReset(ctx context.Context, token string) (string, error)
|
||||
}
|
||||
|
||||
// ErrNoReset is an address that gets no reset link: none has an account, the
|
||||
// account signs in through single sign-on, or it is not active. Never told to
|
||||
// whoever asked, who learns the same either way.
|
||||
var ErrNoReset = errors.New("no reset link for that address")
|
||||
|
||||
// RequestReset mints a reset link token for the person an address belongs
|
||||
// to, when they sign in with a password and are active. A few a quarter of
|
||||
// an hour per address: each sends a mail.
|
||||
func (s *Service) RequestReset(ctx context.Context, rs ResetStore, email string) (string, store.User, error) {
|
||||
email = store.NormalizeEmail(email)
|
||||
if !validEmail(email) {
|
||||
return "", store.User{}, ErrNoReset
|
||||
}
|
||||
if !s.allowAttempt("reset:"+email, 3) {
|
||||
return "", store.User{}, ErrTooManyTries
|
||||
}
|
||||
u, err := rs.UserByEmail(ctx, email)
|
||||
if err != nil || u.PasswordHash == "" || u.State != store.UserActive {
|
||||
return "", store.User{}, ErrNoReset
|
||||
}
|
||||
token, err := rs.CreatePasswordReset(ctx, u.ID, ResetLinkTTL)
|
||||
if err != nil {
|
||||
return "", store.User{}, err
|
||||
}
|
||||
s.log.Info("a password reset link was made", "user", u.ID)
|
||||
return token, u, nil
|
||||
}
|
||||
|
||||
// ResetPassword sets a password from a reset link, which is spent, and signs
|
||||
// the person out everywhere: if they lost it, someone else may have it.
|
||||
func (s *Service) ResetPassword(ctx context.Context, rs ResetStore, pw PasswordChanger, token, next string) error {
|
||||
hash, err := HashPassword(next)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
userID, err := rs.UsePasswordReset(ctx, token)
|
||||
if err != nil {
|
||||
return ErrNoReset
|
||||
}
|
||||
if err := pw.SetPassword(ctx, userID, hash); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := pw.DeleteUserSessions(ctx, userID); err != nil {
|
||||
return err
|
||||
}
|
||||
s.log.Info("a password was set from a reset link", "user", userID)
|
||||
return nil
|
||||
}
|
||||
@@ -323,7 +323,7 @@ func (a *Accountant) Record(res Reservation, u store.Usage) {
|
||||
if u.ServedModel != "" && u.ServedModel != u.Model && a.pricing != nil {
|
||||
price = a.pricing.For(u.ServedModel)
|
||||
}
|
||||
u.CostMicros = price.Cost(u.PromptTokens, u.CompletionTokens)
|
||||
u.CostMicros = price.Cost(u.PromptTokens, u.CompletionTokens, u.Images, u.Meshes)
|
||||
for _, b := range res.Budgets {
|
||||
a.budgets.Add(b.Counter, b.Period, u.CostMicros)
|
||||
}
|
||||
|
||||
@@ -39,17 +39,27 @@ func TestCostRoundsUpSoSmallRequestsAreNotFree(t *testing.T) {
|
||||
// Rounding down would make a stream of tiny requests cost nothing, which is
|
||||
// exactly the traffic shape a quota exists to bound.
|
||||
p := Price{PromptPer1k: 1000, CompletionPer1k: 3000}
|
||||
if got := p.Cost(1, 0); got != 1 {
|
||||
if got := p.Cost(1, 0, 0, 0); got != 1 {
|
||||
t.Errorf("one prompt token cost %d", got)
|
||||
}
|
||||
if got := p.Cost(1000, 1000); got != 4000 {
|
||||
if got := p.Cost(1000, 1000, 0, 0); got != 4000 {
|
||||
t.Errorf("a thousand of each cost %d, want 4000", got)
|
||||
}
|
||||
if got := p.Cost(0, 0); got != 0 {
|
||||
if got := p.Cost(0, 0, 0, 0); got != 0 {
|
||||
t.Errorf("an empty request cost %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPictureCostsItsPriceOnTopOfItsPrompt(t *testing.T) {
|
||||
p := Price{PromptPer1k: 1000, PerImage: 50000}
|
||||
if got := p.Cost(20, 0, 3, 0); got != 20+150000 {
|
||||
t.Errorf("three pictures from a 20-token prompt cost %d", got)
|
||||
}
|
||||
if (Price{PerImage: 1}).Zero() {
|
||||
t.Error("a model priced per picture reads as free, and would pass a spent budget")
|
||||
}
|
||||
}
|
||||
|
||||
func TestZeroPricedModelsStayAvailableWhenTheBudgetIsSpent(t *testing.T) {
|
||||
// This is what makes budget fallback work: a key with nothing left can
|
||||
// still be served something that costs nothing.
|
||||
|
||||
@@ -18,6 +18,13 @@ import (
|
||||
type Price struct {
|
||||
PromptPer1k int64 `koanf:"prompt_per_1k"`
|
||||
CompletionPer1k int64 `koanf:"completion_per_1k"`
|
||||
// PerImage is what one picture costs, for a model that makes them: a
|
||||
// picture's cost is the card's time, which its prompt's few tokens do
|
||||
// not measure.
|
||||
PerImage int64 `koanf:"per_image"`
|
||||
// PerMesh is what one 3D model costs: minutes of a card, which no
|
||||
// token count measures.
|
||||
PerMesh int64 `koanf:"per_mesh"`
|
||||
}
|
||||
|
||||
// Zero reports whether this model spends nothing.
|
||||
@@ -25,14 +32,17 @@ type Price struct {
|
||||
// It matters beyond arithmetic: a zero-cost model stays available to a key
|
||||
// whose budget is spent, which is what makes the plan's budget fallback work
|
||||
// (§7.7).
|
||||
func (p Price) Zero() bool { return p.PromptPer1k == 0 && p.CompletionPer1k == 0 }
|
||||
func (p Price) Zero() bool {
|
||||
return p.PromptPer1k == 0 && p.CompletionPer1k == 0 && p.PerImage == 0 && p.PerMesh == 0
|
||||
}
|
||||
|
||||
// Cost computes what a request spent, rounding up.
|
||||
//
|
||||
// Up rather than to nearest: rounding down would make a stream of tiny requests
|
||||
// free, which is exactly the shape of traffic a quota exists to bound.
|
||||
func (p Price) Cost(promptTokens, completionTokens int64) int64 {
|
||||
return divUp(promptTokens*p.PromptPer1k, 1000) + divUp(completionTokens*p.CompletionPer1k, 1000)
|
||||
func (p Price) Cost(promptTokens, completionTokens, images, meshes int64) int64 {
|
||||
return divUp(promptTokens*p.PromptPer1k, 1000) + divUp(completionTokens*p.CompletionPer1k, 1000) +
|
||||
max(images, 0)*p.PerImage + max(meshes, 0)*p.PerMesh
|
||||
}
|
||||
|
||||
func divUp(n, d int64) int64 {
|
||||
|
||||
+166
-3
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -34,6 +35,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/train"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
|
||||
@@ -136,6 +138,13 @@ type Agent struct {
|
||||
// by mu.
|
||||
kv *kvevents.Mirror
|
||||
kvSubs map[string]*kvevents.Subscriber
|
||||
|
||||
// settingsCheck says whether the cluster's settings for this agent
|
||||
// would load beside its agent.yaml; see SetSettingsCheck.
|
||||
settingsCheck func([]byte) error
|
||||
|
||||
// trainer runs LoRA training jobs, one at a time (BACKLOG F4).
|
||||
trainer *train.Runner
|
||||
}
|
||||
|
||||
// connectHTTPClient is the HTTP client connect uses; an interface so tests can
|
||||
@@ -182,6 +191,18 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
|
||||
kvSubs: map[string]*kvevents.Subscriber{},
|
||||
}
|
||||
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
|
||||
runAs := cfg.Superv.RunAs
|
||||
if runAs == "" {
|
||||
runAs = supervisor.DefaultRunAs
|
||||
}
|
||||
// Where the engines are, as the installer has it when the file says
|
||||
// nothing.
|
||||
enginesDir := cfg.EnginesDir
|
||||
if enginesDir == "" {
|
||||
enginesDir = provision.DefaultRoot
|
||||
}
|
||||
a.trainer = train.New(train.Env{EnginesDir: enginesDir, WorkDir: filepath.Join(cfg.Superv.WorkDir, "trainer"),
|
||||
RunAs: runAs, Log: log})
|
||||
return a, nil
|
||||
}
|
||||
|
||||
@@ -371,6 +392,12 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) {
|
||||
_ = a.sup.Stop(ctx, o.ShardID)
|
||||
return
|
||||
}
|
||||
// What it computes on, for one started before the agent kept that.
|
||||
if a.sup.ComputeOf(o.ShardID) == nil {
|
||||
if c, ok := a.gpuCompute(ctx, o.Spec.GPUIndices, sh.Status().PID); ok {
|
||||
a.sup.SetCompute(o.ShardID, c)
|
||||
}
|
||||
}
|
||||
a.log.Info("adopted engine is serving",
|
||||
"shard", o.ShardID, "replica", o.ReplicaID)
|
||||
}
|
||||
@@ -411,6 +438,17 @@ func (a *Agent) weights() *weights.Fetcher {
|
||||
}
|
||||
|
||||
// runningShards describes what this agent is supervising, for registration.
|
||||
// unready are the shards a registration reported as not yet ready.
|
||||
func unready(running []*llmgwv1.RunningShard) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, r := range running {
|
||||
if r.GetState() != agentsvc.StateToWire(supervisor.Ready) {
|
||||
out[r.GetShardId()] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *Agent) runningShards() []*llmgwv1.RunningShard {
|
||||
var out []*llmgwv1.RunningShard
|
||||
for _, st := range a.sup.Statuses() {
|
||||
@@ -436,6 +474,7 @@ func (a *Agent) runningShards() []*llmgwv1.RunningShard {
|
||||
Pool: spec.Pool,
|
||||
Fabric: spec.Fabric,
|
||||
ProfileFp: spec.ProfileFP,
|
||||
Compute: computeToWire(a.sup.ComputeOf(st.ID)),
|
||||
LeaderAddr: a.cfg.MgmtAddr,
|
||||
LeaderPort: int32(spec.LeaderPort),
|
||||
})
|
||||
@@ -564,8 +603,9 @@ func (a *Agent) session(ctx context.Context, server string) error {
|
||||
a.sendLoop(ctx, stream, out)
|
||||
}()
|
||||
defer wg.Wait()
|
||||
go a.watchExits(ctx, out)
|
||||
go a.watchExits(ctx, out, unready(reg.RunningShards))
|
||||
go a.forwardKV(ctx, out)
|
||||
go a.reportCompute(ctx, out)
|
||||
|
||||
for {
|
||||
msg, err := stream.Receive()
|
||||
@@ -861,6 +901,20 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
|
||||
a.setControlPlanes(m.ControlPlanes)
|
||||
case *llmgwv1.ControlMessage_Firewall:
|
||||
go a.applyFirewall(m.Firewall)
|
||||
case *llmgwv1.ControlMessage_AgentSettings:
|
||||
go a.applySettings(m.AgentSettings)
|
||||
case *llmgwv1.ControlMessage_TrainStart:
|
||||
go a.trainJob(ctx, m.TrainStart, out)
|
||||
case *llmgwv1.ControlMessage_TrainCancel:
|
||||
a.trainer.Cancel(m.TrainCancel.GetJobId())
|
||||
case *llmgwv1.ControlMessage_RestartAgent:
|
||||
go func() {
|
||||
a.log.Info("restarting, as the control plane asks; the engines keep serving",
|
||||
"reason", m.RestartAgent.GetReason())
|
||||
// A moment for the log line to leave before the process does.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
exitForUpgrade()
|
||||
}()
|
||||
case *llmgwv1.ControlMessage_Logs:
|
||||
reply := a.logsReply(m.Logs)
|
||||
// Off the receive loop: a full send queue must not stop commands being
|
||||
@@ -1078,6 +1132,31 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
stop()
|
||||
spec.Profile.ModelPath = path
|
||||
}
|
||||
// A shared ComfyUI replica's members' weights, each fetched like the
|
||||
// model's, and handed to the gate where they landed.
|
||||
if raw := spec.Profile.Env[engines.SharedModelsEnv]; raw != "" {
|
||||
var ms []map[string]string
|
||||
if err := json.Unmarshal([]byte(raw), &ms); err != nil {
|
||||
report(supervisor.Failed, "the shared replica's models are not readable: "+err.Error(), nil)
|
||||
return
|
||||
}
|
||||
for i, m := range ms {
|
||||
stop := a.watchFetch(m["source"], func(p weights.Progress) {
|
||||
send(&llmgwv1.ShardStatus{State: agentsvc.StateToWire(supervisor.Starting),
|
||||
Fetch: &llmgwv1.FetchProgress{Source: p.Source, BytesDone: p.BytesDone, BytesTotal: p.BytesTotal}})
|
||||
})
|
||||
path, err := a.weights().Ensure(ctx, m["source"])
|
||||
stop()
|
||||
if err != nil {
|
||||
report(supervisor.Failed, fmt.Sprintf("model %s: %v", m["name"], err), nil)
|
||||
return
|
||||
}
|
||||
ms[i]["path"] = path
|
||||
}
|
||||
b, _ := json.Marshal(ms)
|
||||
spec.Profile.Env = maps.Clone(spec.Profile.Env)
|
||||
spec.Profile.Env[engines.SharedModelsEnv] = string(b)
|
||||
}
|
||||
// Adapters are fetched like the model, and before the engine starts: it
|
||||
// loads them at start, and one missing is a replica that answers for the
|
||||
// base model under the adapter's name.
|
||||
@@ -1130,6 +1209,14 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
} else {
|
||||
err = a.sup.WaitAlive(ctx, sh)
|
||||
}
|
||||
// Where it put the model, by its own account: said with the readiness,
|
||||
// and a GPU placement it computes on the CPU refused even where
|
||||
// nvidia-smi cannot be asked.
|
||||
compute, said, cerr := engineCompute(eng, spec.GPUIndices, sh.Logs(512))
|
||||
if err == nil && cerr != nil {
|
||||
err = cerr
|
||||
_ = a.sup.Stop(ctx, shardID)
|
||||
}
|
||||
if err == nil {
|
||||
err = a.onItsGPUs(ctx, eng, spec, sh.Status().PID)
|
||||
if err != nil {
|
||||
@@ -1142,7 +1229,19 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
report(supervisor.Failed, err.Error(), sh.Logs(20))
|
||||
return
|
||||
}
|
||||
report(supervisor.Ready, "", nil)
|
||||
// The engine's own account when it gave one, with its layers; else what
|
||||
// nvidia-smi sees the shard holding.
|
||||
if !said {
|
||||
compute, said = a.gpuCompute(ctx, spec.GPUIndices, sh.Status().PID)
|
||||
}
|
||||
if said {
|
||||
// Kept with the shard, so an agent restarted, or a control plane
|
||||
// taking over, still learns it.
|
||||
a.sup.SetCompute(shardID, compute)
|
||||
send(&llmgwv1.ShardStatus{State: agentsvc.StateToWire(supervisor.Ready), Compute: computeToWire(&compute)})
|
||||
} else {
|
||||
report(supervisor.Ready, "", nil)
|
||||
}
|
||||
// Its dying later is reported by the session's watchExits.
|
||||
}
|
||||
|
||||
@@ -1155,7 +1254,14 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
// reported the engine dying; and an adopted engine had no watcher at all —
|
||||
// on node06 one was killed and its replica stayed "ready" for the control
|
||||
// plane, routed to and failing, indefinitely.
|
||||
func (a *Agent) watchExits(ctx context.Context, out chan<- *llmgwv1.AgentMessage) {
|
||||
//
|
||||
// It also reports the shards the registration said were not ready yet once
|
||||
// they are. An adopted engine is health-checked after the agent registers,
|
||||
// and a shard started on the stream before is reported on that stream; so
|
||||
// when the registration came first nothing ever told the control plane, and
|
||||
// the replica stayed "starting" while it served — on node05, after a
|
||||
// control plane restart, the only replica of a model.
|
||||
func (a *Agent) watchExits(ctx context.Context, out chan<- *llmgwv1.AgentMessage, pending map[string]bool) {
|
||||
t := time.NewTicker(time.Second)
|
||||
defer t.Stop()
|
||||
ready := map[string]bool{}
|
||||
@@ -1164,6 +1270,24 @@ func (a *Agent) watchExits(ctx context.Context, out chan<- *llmgwv1.AgentMessage
|
||||
switch st.State {
|
||||
case supervisor.Ready:
|
||||
ready[st.ID] = true
|
||||
if !pending[st.ID] {
|
||||
continue
|
||||
}
|
||||
delete(pending, st.ID)
|
||||
sh, err := a.sup.Get(st.ID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||
NodeId: a.cfg.NodeID, ReplicaId: sh.Spec.ReplicaID, ShardId: st.ID,
|
||||
State: agentsvc.StateToWire(st.State),
|
||||
Compute: computeToWire(a.sup.ComputeOf(st.ID)),
|
||||
}}}
|
||||
select {
|
||||
case out <- msg:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
case supervisor.Failed:
|
||||
if !ready[st.ID] {
|
||||
continue // never ready here: its start reported it
|
||||
@@ -1373,3 +1497,42 @@ func ResolveManagement(c Config) (addr, iface string) {
|
||||
}
|
||||
return addr, iface
|
||||
}
|
||||
|
||||
// reportCompute tells the control plane, once a session, what each running
|
||||
// shard computes on as soon as that is known: an adopted engine's is found
|
||||
// after registration, which carried what was known then.
|
||||
func (a *Agent) reportCompute(ctx context.Context, out chan<- *llmgwv1.AgentMessage) {
|
||||
sent := map[string]bool{}
|
||||
t := time.NewTicker(5 * time.Second)
|
||||
defer t.Stop()
|
||||
for range 60 {
|
||||
for _, st := range a.sup.Statuses() {
|
||||
if st.State.Terminal() || sent[st.ID] {
|
||||
continue
|
||||
}
|
||||
c := a.sup.ComputeOf(st.ID)
|
||||
if c == nil {
|
||||
continue
|
||||
}
|
||||
sh, err := a.sup.Get(st.ID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// No state: a report of compute alone, not a transition.
|
||||
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||
NodeId: a.cfg.NodeID, ReplicaId: sh.Spec.ReplicaID, ShardId: st.ID, Compute: computeToWire(c),
|
||||
}}}
|
||||
select {
|
||||
case out <- msg:
|
||||
sent[st.ID] = true
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -37,11 +37,12 @@ import (
|
||||
type statusRecorder struct {
|
||||
ch chan status
|
||||
|
||||
mu sync.Mutex
|
||||
calls []adoptCall
|
||||
adopted []reconciler.AdoptedShard
|
||||
unclaim []string // shard ids to report back as unclaimed
|
||||
fetches []fetchReport
|
||||
mu sync.Mutex
|
||||
calls []adoptCall
|
||||
adopted []reconciler.AdoptedShard
|
||||
unclaim []string // shard ids to report back as unclaimed
|
||||
fetches []fetchReport
|
||||
computes []reconciler.Compute
|
||||
}
|
||||
|
||||
type status struct {
|
||||
@@ -61,6 +62,12 @@ func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor.
|
||||
|
||||
// fetches records download progress, so a test can assert that a fetch is
|
||||
// visible from the control plane rather than only in the node's log.
|
||||
func (r *statusRecorder) OnShardCompute(replicaID, shardID string, c reconciler.Compute) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.computes = append(r.computes, c)
|
||||
}
|
||||
|
||||
func (r *statusRecorder) OnFetchProgress(replicaID, shardID, source string, done, total int64) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
@@ -103,6 +103,11 @@ type Config struct {
|
||||
Tracing obs.Tracing `koanf:"tracing"`
|
||||
|
||||
Version string
|
||||
|
||||
// Settings is the cluster's settings for this agent it was started on,
|
||||
// as the control plane sent them: what a newly sent version is
|
||||
// compared with.
|
||||
Settings string `koanf:"-"`
|
||||
}
|
||||
|
||||
// DefaultConfig returns sensible defaults.
|
||||
|
||||
@@ -3,6 +3,7 @@ package discovery
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -142,17 +143,29 @@ func cpuFlagsFrom(path string) []string {
|
||||
// nvidiaDriverFrom reads the loaded kernel module's version, which exists
|
||||
// exactly when a driver is loaded — unlike nvidia-smi, which can be installed
|
||||
// on a machine whose module failed to load.
|
||||
var driverVersion = regexp.MustCompile(`^[0-9]+\.[0-9]+(\.[0-9]+)?$`)
|
||||
|
||||
func nvidiaDriverFrom(path string) string {
|
||||
b, err := os.ReadFile(path) //nolint:gosec // a fixed system path
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
// "NVRM version: NVIDIA UNIX x86_64 Kernel Module 570.86.15 Thu ..."
|
||||
// "NVRM version: NVIDIA UNIX x86_64 Kernel Module 570.86.15 Thu ...", or
|
||||
// for the open module, the default since 560, "NVIDIA UNIX Open Kernel
|
||||
// Module for x86_64 580.82.07 Release Build ...": the first version
|
||||
// after "Module", not the word after it. Read as "for", the driver was
|
||||
// unknown, and every build was tried whatever driver it needs.
|
||||
fields := strings.Fields(string(b))
|
||||
for i, f := range fields {
|
||||
if f == "Module" && i+1 < len(fields) {
|
||||
return fields[i+1]
|
||||
if f != "Module" {
|
||||
continue
|
||||
}
|
||||
for _, v := range fields[i+1:] {
|
||||
if driverVersion.MatchString(v) {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNVIDIADriverFrom(t *testing.T) {
|
||||
for _, tc := range []struct{ text, want string }{
|
||||
{"NVRM version: NVIDIA UNIX x86_64 Kernel Module 570.86.15 Thu Jan 23 21:30:38 UTC 2025\nGCC version: gcc version 12.2.0\n", "570.86.15"},
|
||||
{"NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 580.82.07 Release Build (dvs-builder@U16-I3-B03-4-3) Wed Aug 27 00:51:12 UTC 2025\n", "580.82.07"},
|
||||
{"NVRM version: NVIDIA UNIX Open Kernel Module for aarch64 580.95.05 Release Build\n", "580.95.05"},
|
||||
{"NVRM version: NVIDIA UNIX x86_64 Kernel Module 535.54 Mon\n", "535.54"},
|
||||
{"something else\n", ""},
|
||||
} {
|
||||
p := filepath.Join(t.TempDir(), "version")
|
||||
if err := os.WriteFile(p, []byte(tc.text), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := nvidiaDriverFrom(p); got != tc.want {
|
||||
t.Errorf("%q: got %q, want %q", tc.text, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// ComfyUIConfig locates a ComfyUI installation.
|
||||
type ComfyUIConfig struct {
|
||||
// Dir is ComfyUI's directory, where main.py is. Readable by the user
|
||||
// engines run as.
|
||||
Dir string `koanf:"dir"`
|
||||
// Python is the interpreter of its environment; Dir's .venv when unset.
|
||||
Python string `koanf:"python"`
|
||||
}
|
||||
|
||||
// ComfyUI runs ComfyUI as an engine, behind llmgw's gate (llmgw
|
||||
// comfyui-gate): ComfyUI asks for no key, so it listens on the loopback and
|
||||
// the gate, which does, on the replica's port. Its replicas make pictures
|
||||
// for /v1/images/generations.
|
||||
type ComfyUI struct {
|
||||
cfg ComfyUIConfig
|
||||
installed atomic.Pointer[string]
|
||||
}
|
||||
|
||||
// Name identifies the engine in the catalog.
|
||||
func (*ComfyUI) Name() string { return "comfyui" }
|
||||
|
||||
// HealthPath is the gate's, which answers once ComfyUI does.
|
||||
func (*ComfyUI) HealthPath() string { return "/health" }
|
||||
|
||||
// MetricsPath is empty: ComfyUI publishes no metrics, and the router reads
|
||||
// none from it.
|
||||
func (*ComfyUI) MetricsPath() string { return "" }
|
||||
|
||||
// ServesHTTP is true: a ComfyUI replica is one process on one node.
|
||||
func (*ComfyUI) ServesHTTP(ShardSpec) bool { return true }
|
||||
|
||||
// UseInstalled points the engine at the environment the agent installed:
|
||||
// its interpreter, with ComfyUI's code beside its bin, in comfyui/.
|
||||
func (c *ComfyUI) UseInstalled(python string) { c.installed.Store(&python) }
|
||||
|
||||
// Configured reports whether the configuration names a ComfyUI.
|
||||
func (c *ComfyUI) Configured() bool { return c.cfg.Dir != "" }
|
||||
|
||||
func (c *ComfyUI) dir() string {
|
||||
if p := c.installed.Load(); p != nil {
|
||||
return filepath.Join(filepath.Dir(filepath.Dir(*p)), "comfyui")
|
||||
}
|
||||
return c.cfg.Dir
|
||||
}
|
||||
|
||||
func (c *ComfyUI) python() string {
|
||||
if p := c.installed.Load(); p != nil {
|
||||
return *p
|
||||
}
|
||||
if c.cfg.Python != "" {
|
||||
return c.cfg.Python
|
||||
}
|
||||
return filepath.Join(c.cfg.Dir, ".venv", "bin", "python")
|
||||
}
|
||||
|
||||
var comfyVersion = regexp.MustCompile(`__version__\s*=\s*"([^"]+)"`)
|
||||
|
||||
// Probe reports ComfyUI's version, from the file its build writes it to:
|
||||
// importing ComfyUI to ask would load torch and look for cards.
|
||||
func (c *ComfyUI) Probe(context.Context) (string, error) {
|
||||
dir := c.dir()
|
||||
if dir == "" {
|
||||
return "", fmt.Errorf("no ComfyUI directory is configured")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "main.py")); err != nil {
|
||||
return "", fmt.Errorf("%s is not ComfyUI: %w", dir, err)
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(dir, "comfyui_version.py"))
|
||||
if err != nil {
|
||||
return Unknown, nil //nolint:nilerr // installed, of a version it does not say
|
||||
}
|
||||
if m := comfyVersion.FindSubmatch(b); m != nil {
|
||||
return string(m[1]), nil
|
||||
}
|
||||
return Unknown, nil
|
||||
}
|
||||
|
||||
// Validate checks the profile: one card, on one node. ComfyUI runs a
|
||||
// workflow on one device and has no tensor parallelism to spread it by.
|
||||
func (*ComfyUI) Validate(p Profile, gpus []GPUInfo) error {
|
||||
if p.Nodes > 1 || p.TP > 1 || p.PP > 1 || p.GPUsPerNode > 1 {
|
||||
return fmt.Errorf("profile %q: ComfyUI runs on one GPU of one node", p.Name)
|
||||
}
|
||||
return validateCommon(p, gpus)
|
||||
}
|
||||
|
||||
// Commands is the gate, which starts ComfyUI itself.
|
||||
func (c *ComfyUI) Commands(s ShardSpec) ([]Command, error) {
|
||||
dir := c.dir()
|
||||
if dir == "" {
|
||||
return nil, fmt.Errorf("comfyui: no ComfyUI directory is configured (engines.comfyui.dir)")
|
||||
}
|
||||
bin, err := os.Executable()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p := s.Profile
|
||||
args := []string{"comfyui-gate",
|
||||
"--listen", net.JoinHostPort(s.Host(), strconv.Itoa(s.HTTPPort)),
|
||||
"--model", p.ServedModelName,
|
||||
"--weights", p.ModelPath,
|
||||
// Relative to where engines run — their user's home — so it is
|
||||
// theirs to write.
|
||||
"--state", filepath.Join("comfyui", s.ReplicaID),
|
||||
"--python", c.python(),
|
||||
"--comfyui", dir,
|
||||
"--",
|
||||
}
|
||||
args = append(args, p.Args...)
|
||||
env := baseEnv(s)
|
||||
if s.APIKey != "" {
|
||||
env["LLMGW_ENGINE_KEY"] = s.APIKey
|
||||
}
|
||||
// A model's custom nodes and the repository a node loads its weights by,
|
||||
// which only the catalog sets — a profile's env may not name them — and
|
||||
// so taken before baseEnv drops them. Without the repository the gate
|
||||
// links no Hugging Face cache, and sf3d's node went to the network for
|
||||
// its gated repository and was refused with a 401.
|
||||
for _, k := range []string{comfyNodesEnv, comfyHFRepoEnv, SharedModelsEnv} {
|
||||
if v := p.Env[k]; v != "" {
|
||||
env[k] = v
|
||||
}
|
||||
}
|
||||
return []Command{{Name: "comfyui", Path: bin, Args: args, Env: materializeEnv(env)}}, nil
|
||||
}
|
||||
|
||||
// comfyNodesEnv is the custom nodes the catalog gives a ComfyUI model, as
|
||||
// JSON; catalog.NodesEnv and comfy.NodesEnv by the same name.
|
||||
const comfyNodesEnv = "LLMGW_COMFYUI_NODES"
|
||||
|
||||
// comfyHFRepoEnv is the repository a custom node loads the weights by name
|
||||
// from; catalog.HFRepoEnv and comfy.HFRepoEnv by the same name.
|
||||
const comfyHFRepoEnv = "LLMGW_COMFYUI_HF_REPO"
|
||||
|
||||
// SharedModelsEnv is a shared replica's members — name, source, and the
|
||||
// path the agent fetched it to — as JSON; catalog.SharedModelsEnv and
|
||||
// comfy.SharedModelsEnv by the same name.
|
||||
const SharedModelsEnv = "LLMGW_COMFYUI_MODELS"
|
||||
@@ -0,0 +1,99 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestComfyUIRunsBehindTheGateWithItsKeyOutOfSight(t *testing.T) {
|
||||
c := &ComfyUI{cfg: ComfyUIConfig{Dir: "/opt/comfyui"}}
|
||||
cmds, err := c.Commands(ShardSpec{
|
||||
ReplicaID: "r-flux-1", HTTPPort: 8100, BindAddr: "10.0.0.147", APIKey: "sekret", GPUIndices: []int{0},
|
||||
Profile: Profile{Name: "p", Engine: "comfyui", ModelPath: "/w/flux.safetensors", ServedModelName: "flux",
|
||||
Args: []string{"--disable-smart-memory"}},
|
||||
})
|
||||
if err != nil || len(cmds) != 1 {
|
||||
t.Fatalf("%v %v", cmds, err)
|
||||
}
|
||||
args := strings.Join(cmds[0].Args, " ")
|
||||
for _, want := range []string{"comfyui-gate", "--listen 10.0.0.147:8100", "--model flux", "--weights /w/flux.safetensors",
|
||||
"--state comfyui/r-flux-1", "--python /opt/comfyui/.venv/bin/python", "--comfyui /opt/comfyui", "-- --disable-smart-memory"} {
|
||||
if !strings.Contains(args, want) {
|
||||
t.Errorf("%q not in %q", want, args)
|
||||
}
|
||||
}
|
||||
if strings.Contains(args, "sekret") {
|
||||
t.Error("the key is on the command line, where every user can read it")
|
||||
}
|
||||
if !slices.Contains(cmds[0].Env, "LLMGW_ENGINE_KEY=sekret") || !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=0") {
|
||||
t.Error("the key or the card is not in the environment")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheGateIsToldTheNodesAndTheRepositoryTheCatalogGives(t *testing.T) {
|
||||
// The repository was dropped on the way: sf3d's node went to the network
|
||||
// for its gated repository, with no cache linked, and got a 401.
|
||||
c := &ComfyUI{cfg: ComfyUIConfig{Dir: "/opt/comfyui"}}
|
||||
cmds, err := c.Commands(ShardSpec{
|
||||
ReplicaID: "r-sf3d-1", HTTPPort: 8100, BindAddr: "10.0.0.147", GPUIndices: []int{0},
|
||||
Profile: Profile{Name: "p", Engine: "comfyui", ModelPath: "/w/sf3d/", ServedModelName: "sf3d",
|
||||
Env: map[string]string{comfyNodesEnv: `[{"repo":"x"}]`, comfyHFRepoEnv: "stabilityai/stable-fast-3d"}},
|
||||
})
|
||||
if err != nil || len(cmds) != 1 {
|
||||
t.Fatalf("%v %v", cmds, err)
|
||||
}
|
||||
for _, want := range []string{comfyNodesEnv + `=[{"repo":"x"}]`, comfyHFRepoEnv + "=stabilityai/stable-fast-3d"} {
|
||||
if !slices.Contains(cmds[0].Env, want) {
|
||||
t.Errorf("%q not in the gate's environment", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestComfyUIRunsOnOneCard(t *testing.T) {
|
||||
c := &ComfyUI{}
|
||||
if err := c.Validate(Profile{Name: "p", ModelPath: "/w", TP: 2}, h100(2)); err == nil {
|
||||
t.Error("a two-card ComfyUI profile was accepted")
|
||||
}
|
||||
if err := c.Validate(Profile{Name: "p", ModelPath: "/w", TP: 1}, h100(1)); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if _, err := c.Commands(ShardSpec{Profile: Profile{ModelPath: "/w"}}); err == nil {
|
||||
t.Error("commands without a ComfyUI directory")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComfyUIIsOfferedByTheVersionItsBuildWrote(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
c := &ComfyUI{cfg: ComfyUIConfig{Dir: dir}}
|
||||
if _, err := c.Probe(t.Context()); err == nil {
|
||||
t.Error("a directory without main.py passed for ComfyUI")
|
||||
}
|
||||
_ = os.WriteFile(filepath.Join(dir, "main.py"), nil, 0o644)
|
||||
_ = os.WriteFile(filepath.Join(dir, "comfyui_version.py"), []byte("__version__ = \"0.39.2\"\n"), 0o644)
|
||||
if v, err := c.Probe(t.Context()); err != nil || v != "0.39.2" {
|
||||
t.Errorf("%q %v", v, err)
|
||||
}
|
||||
if got := NewRegistry(Config{ComfyUI: ComfyUIConfig{Dir: dir}}).ProbeConfigured(t.Context()); got["comfyui"] != "0.39.2" {
|
||||
t.Errorf("a configured ComfyUI is reported as %v", got)
|
||||
}
|
||||
if err := UnsafeArgs("comfyui", []string{"--enable-manager"}); err == nil {
|
||||
t.Error("a profile may install ComfyUI nodes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnInstalledComfyUIIsFoundBesideItsInterpreter(t *testing.T) {
|
||||
c := &ComfyUI{cfg: ComfyUIConfig{Dir: "/opt/by-hand"}}
|
||||
c.UseInstalled("/opt/llmgw/engines/comfyui/0.39.2-cu13/bin/python")
|
||||
cmds, err := c.Commands(ShardSpec{ReplicaID: "r", HTTPPort: 1, Profile: Profile{ModelPath: "/w", ServedModelName: "m"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
args := strings.Join(cmds[0].Args, " ")
|
||||
if !strings.Contains(args, "--python /opt/llmgw/engines/comfyui/0.39.2-cu13/bin/python") ||
|
||||
!strings.Contains(args, "--comfyui /opt/llmgw/engines/comfyui/0.39.2-cu13/comfyui") {
|
||||
t.Errorf("the installed ComfyUI is not what runs: %s", args)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Compute is where an engine runs its model, as it said while loading it.
|
||||
type Compute struct {
|
||||
// Backend is CUDA, Vulkan, Metal, ROCm or CPU.
|
||||
Backend string
|
||||
// Devices are those it named: "CUDA0: Quadro RTX 6000 (7.5)".
|
||||
Devices []string
|
||||
// Offloaded of Total layers went to the devices; 0 and 0 when it did
|
||||
// not say.
|
||||
Offloaded, Total int
|
||||
// NoGPU is the engine saying it found no device it could use.
|
||||
NoGPU bool
|
||||
}
|
||||
|
||||
var (
|
||||
offloadedRE = regexp.MustCompile(`offloaded (\d+)/(\d+) layers to GPU`)
|
||||
usingRE = regexp.MustCompile(`using device ([A-Za-z]+\d+) \(([^)]*)\)`)
|
||||
ccRE = regexp.MustCompile(`Device (\d+): ([^,]+), compute capability ([0-9.]+)`)
|
||||
noDevicesRE = regexp.MustCompile(`found 0 (CUDA|Vulkan|ROCm) devices|no usable GPU found`)
|
||||
)
|
||||
|
||||
// ReadLlamaCompute reads llama.cpp's start-up output for where it put the
|
||||
// model, reporting false when the output says nothing of it.
|
||||
func ReadLlamaCompute(lines []string) (Compute, bool) {
|
||||
var c Compute
|
||||
known := false
|
||||
cc := map[string]string{} // device index → compute capability
|
||||
for _, l := range lines {
|
||||
if m := ccRE.FindStringSubmatch(l); m != nil {
|
||||
cc[m[1]] = m[3]
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, l := range lines {
|
||||
if m := offloadedRE.FindStringSubmatch(l); m != nil {
|
||||
c.Offloaded, _ = strconv.Atoi(m[1])
|
||||
c.Total, _ = strconv.Atoi(m[2])
|
||||
known = true
|
||||
}
|
||||
if m := usingRE.FindStringSubmatch(l); m != nil && !seen[m[1]] {
|
||||
seen[m[1]] = true
|
||||
d := m[1] + ": " + strings.TrimSpace(m[2])
|
||||
idx := strings.TrimLeft(m[1], "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz")
|
||||
if v, ok := cc[idx]; ok && strings.HasPrefix(m[1], "CUDA") {
|
||||
d += " (" + v + ")"
|
||||
}
|
||||
c.Devices = append(c.Devices, d)
|
||||
if c.Backend == "" {
|
||||
c.Backend = strings.TrimRight(m[1], "0123456789")
|
||||
}
|
||||
known = true
|
||||
}
|
||||
if noDevicesRE.MatchString(l) {
|
||||
c.NoGPU, known = true, true
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case c.OnTheCPU():
|
||||
c.Backend = "CPU"
|
||||
case c.Backend == "" && c.Offloaded > 0:
|
||||
c.Backend = "GPU"
|
||||
case c.Backend == "":
|
||||
c.Backend = "CPU"
|
||||
}
|
||||
return c, known
|
||||
}
|
||||
|
||||
// OnTheCPU is whether a model the engine was given GPUs for is computed on
|
||||
// the CPU, by its own account: no device found, or no layer offloaded.
|
||||
func (c Compute) OnTheCPU() bool {
|
||||
return c.NoGPU || (c.Total > 0 && c.Offloaded == 0)
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLlamaCppSaysWhereItPutTheModel(t *testing.T) {
|
||||
gpu := strings.Split(`ggml_cuda_init: found 1 CUDA devices:
|
||||
Device 0: Quadro RTX 6000, compute capability 7.5, VMM: yes
|
||||
load_backend: loaded CUDA backend from /opt/llmgw/engines/llamacpp/b10991/libggml-cuda.so
|
||||
llama_model_load_from_file_impl: using device CUDA0 (Quadro RTX 6000) - 23800 MiB free
|
||||
load_tensors: offloading 48 repeating layers to GPU
|
||||
load_tensors: offloaded 49/49 layers to GPU`, "\n")
|
||||
c, ok := ReadLlamaCompute(gpu)
|
||||
if !ok || c.Backend != "CUDA" || c.Offloaded != 49 || c.Total != 49 || c.OnTheCPU() ||
|
||||
len(c.Devices) != 1 || c.Devices[0] != "CUDA0: Quadro RTX 6000 (7.5)" {
|
||||
t.Errorf("on the GPU: %+v %v", c, ok)
|
||||
}
|
||||
|
||||
cpu := strings.Split(`warning: no usable GPU found, --gpu-layers option will be ignored
|
||||
warning: one possible reason is that llama.cpp was compiled without GPU support
|
||||
load_tensors: offloaded 0/49 layers to GPU`, "\n")
|
||||
c, ok = ReadLlamaCompute(cpu)
|
||||
if !ok || c.Backend != "CPU" || !c.OnTheCPU() {
|
||||
t.Errorf("fell back to the CPU: %+v %v", c, ok)
|
||||
}
|
||||
|
||||
if _, ok := ReadLlamaCompute([]string{"main: server is listening on http://127.0.0.1:20001"}); ok {
|
||||
t.Error("output that says nothing was read as saying something")
|
||||
}
|
||||
}
|
||||
@@ -114,6 +114,10 @@ type ShardSpec struct {
|
||||
// lets the control plane tell, after its own restart, whether a shard that
|
||||
// is still running matches the profile the catalog holds now.
|
||||
ProfileFP string
|
||||
|
||||
// Compute is where the engine said it put the model as it became ready,
|
||||
// kept with the shard's state so an agent restarted later still knows.
|
||||
Compute *Compute `json:",omitempty"`
|
||||
}
|
||||
|
||||
// Host is where the engine listens.
|
||||
@@ -210,6 +214,7 @@ func NewRegistry(cfg Config) *Registry {
|
||||
r.Add(&LlamaCpp{cfg: cfg.LlamaCpp})
|
||||
r.Add(&TRTLLM{cfg: cfg.TRTLLM})
|
||||
r.Add(&Sim{cfg: cfg.Sim})
|
||||
r.Add(&ComfyUI{cfg: cfg.ComfyUI})
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -258,6 +263,8 @@ type Config struct {
|
||||
// offers it only when its interpreter is named here.
|
||||
TRTLLM TRTLLMConfig `koanf:"trtllm"`
|
||||
Sim SimConfig `koanf:"sim"`
|
||||
// ComfyUI makes pictures; offered by a node whose ComfyUI is named here.
|
||||
ComfyUI ComfyUIConfig `koanf:"comfyui"`
|
||||
}
|
||||
|
||||
// --- shared helpers ---
|
||||
|
||||
@@ -477,7 +477,7 @@ func TestValidateRequiresModelPath(t *testing.T) {
|
||||
|
||||
func TestRegistry(t *testing.T) {
|
||||
r := NewRegistry(Config{})
|
||||
if got := r.Names(); !slices.Equal(got, []string{"llamacpp", "sglang", "sim", "trtllm", "vllm"}) {
|
||||
if got := r.Names(); !slices.Equal(got, []string{"comfyui", "llamacpp", "sglang", "sim", "trtllm", "vllm"}) {
|
||||
t.Errorf("registry contents: %v", got)
|
||||
}
|
||||
if _, err := r.Get("vllm"); err != nil {
|
||||
|
||||
@@ -277,6 +277,9 @@ func (l *LlamaCpp) bin() string {
|
||||
return "llama-server"
|
||||
}
|
||||
|
||||
// Binary is the llama-server this engine runs.
|
||||
func (l *LlamaCpp) Binary() string { return l.bin() }
|
||||
|
||||
// FallsBackToCPU says that llama.cpp keeps running when its GPU backend does
|
||||
// not load: --n-gpu-layers asks for layers on a GPU it then has none of, and
|
||||
// the server comes up and answers on the CPU, at a tenth of the speed, with no
|
||||
|
||||
@@ -119,6 +119,19 @@ var flagsNever = map[string]map[string]string{
|
||||
"--model-path": "is the model's source, set by the agent", "--served-model-name": "is set by the agent",
|
||||
"--lora-paths": "loads files", "--download-dir": "writes files",
|
||||
},
|
||||
// ComfyUI's main.py, behind the gate that sets where it listens and
|
||||
// where its files are.
|
||||
"comfyui": {
|
||||
"--listen": "is set by the agent", "--base-directory": "is set by the agent",
|
||||
"--extra-model-paths-config": "is set by the agent",
|
||||
"--output-directory": "writes files", "--temp-directory": "writes files",
|
||||
"--input-directory": "reads files", "--user-directory": "writes files",
|
||||
"--database-url": "writes a database elsewhere", "--front-end-root": "serves a directory",
|
||||
"--enable-cors-header": "lets other sites' pages call it",
|
||||
"--enable-manager": "installs nodes, which run Python",
|
||||
"--tls-keyfile": "reads a file", "--tls-certfile": "reads a file",
|
||||
"--comfy-api-base": "sends work to another service",
|
||||
},
|
||||
// trtllm-serve spells its flags with underscores, so the shared list's
|
||||
// hyphenated names do not reach them.
|
||||
"trtllm": {
|
||||
|
||||
@@ -4,10 +4,12 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
)
|
||||
@@ -153,3 +155,70 @@ func joinIndices(is []int) string {
|
||||
}
|
||||
return strings.Join(s, ", ")
|
||||
}
|
||||
|
||||
// engineCompute is where an engine said it put its model, from its output
|
||||
// so far — llama.cpp's, which says — and false for one that said nothing. A
|
||||
// shard given GPUs that its engine says it computes on the CPU is an error,
|
||||
// found this way even where nvidia-smi cannot be asked.
|
||||
func engineCompute(eng engines.Engine, gpus []int, lines []string) (engines.Compute, bool, error) {
|
||||
if _, ok := eng.(engines.CPUFallback); !ok {
|
||||
return engines.Compute{}, false, nil
|
||||
}
|
||||
read, ok := engines.ReadLlamaCompute(lines)
|
||||
if !ok {
|
||||
return engines.Compute{}, false, nil
|
||||
}
|
||||
if len(gpus) > 0 && read.OnTheCPU() {
|
||||
why := "its GPU backend saw no device it could use"
|
||||
if read.Total > 0 && !read.NoGPU {
|
||||
why = fmt.Sprintf("it offloaded %d of %d layers", read.Offloaded, read.Total)
|
||||
}
|
||||
return read, true, fmt.Errorf("placed on GPU %v but computing on the CPU: %s, by llama.cpp's own account "+
|
||||
"(llmgw ctl engine-check on this node says why)", gpus, why)
|
||||
}
|
||||
return read, true, nil
|
||||
}
|
||||
|
||||
// computeToWire is a compute as it travels to the control plane.
|
||||
func computeToWire(c *engines.Compute) *llmgwv1.Compute {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
return &llmgwv1.Compute{Backend: c.Backend, Devices: c.Devices,
|
||||
LayersOffloaded: int32(c.Offloaded), LayersTotal: int32(c.Total)} //nolint:gosec // layer counts
|
||||
}
|
||||
|
||||
// gpuCompute is where a shard computes as nvidia-smi sees it: the CPU for a
|
||||
// shard given no GPU, else the GPUs whose memory its processes hold, and how
|
||||
// much. False when nvidia-smi cannot be asked. Engine-agnostic, and needing
|
||||
// no log line: llama.cpp says where its layers went only at -lv 4.
|
||||
func (a *Agent) gpuCompute(ctx context.Context, gpus []int, pid int) (engines.Compute, bool) {
|
||||
if len(gpus) == 0 {
|
||||
return engines.Compute{Backend: "CPU"}, true
|
||||
}
|
||||
mem, err := gpuMemory(ctx, a.cfg.Discovery.NvidiaSMI)
|
||||
if err != nil || mem == nil || pid <= 0 {
|
||||
return engines.Compute{}, false
|
||||
}
|
||||
group := processGroup(pid)
|
||||
c := engines.Compute{Backend: "CUDA"}
|
||||
for _, g := range mem {
|
||||
if !slices.Contains(gpus, g.Index) {
|
||||
continue
|
||||
}
|
||||
used := 0
|
||||
for _, p := range g.Procs {
|
||||
if holds([]int{p.PID}, pid, group) {
|
||||
used += p.UsedMB
|
||||
}
|
||||
}
|
||||
if used > 0 {
|
||||
c.Devices = append(c.Devices, fmt.Sprintf("GPU %d: %.1f GB", g.Index, float64(used)/1024))
|
||||
}
|
||||
}
|
||||
if len(c.Devices) == 0 {
|
||||
// Placed on GPUs and holding none of their memory.
|
||||
c.Backend = "CPU"
|
||||
}
|
||||
return c, true
|
||||
}
|
||||
@@ -122,3 +122,55 @@ func TestForeignGPUMemoryLeavesOutTheAgentsOwnShards(t *testing.T) {
|
||||
t.Errorf("no reading came back as %v", foreign)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLlamaCppsOwnWordOnTheCPUIsEnoughToRefuse(t *testing.T) {
|
||||
eng := fallbackEngine{}
|
||||
// Without nvidia-smi to ask, the engine's own output says it.
|
||||
c, said, err := engineCompute(eng, []int{0}, []string{
|
||||
"warning: no usable GPU found, --gpu-layers option will be ignored",
|
||||
"load_tensors: offloaded 0/49 layers to GPU"})
|
||||
if err == nil || !said || !strings.Contains(err.Error(), "computing on the CPU") || c.Backend != "CPU" {
|
||||
t.Fatalf("a GPU shard llama.cpp put on the CPU: %v %+v", err, c)
|
||||
}
|
||||
c, said, err = engineCompute(eng, []int{0}, []string{
|
||||
"llama_model_load_from_file_impl: using device CUDA0 (Quadro RTX 6000) - 23800 MiB free",
|
||||
"load_tensors: offloaded 49/49 layers to GPU"})
|
||||
if err != nil || !said || c.Backend != "CUDA" || c.Offloaded != 49 || computeToWire(&c).GetLayersOffloaded() != 49 {
|
||||
t.Errorf("on its GPU: %v %+v", err, c)
|
||||
}
|
||||
// A CPU shard is not refused for being on the CPU, and says so.
|
||||
if c, _, err := engineCompute(eng, nil, []string{"load_tensors: offloaded 0/49 layers to GPU"}); err != nil || c.Backend != "CPU" {
|
||||
t.Errorf("a CPU shard: %v %+v", err, c)
|
||||
}
|
||||
// An engine that said nothing says nothing.
|
||||
if _, said, err := engineCompute(eng, []int{0}, []string{"listening"}); err != nil || said {
|
||||
t.Errorf("nothing said: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNvidiaSMISaysWhereAShardComputes(t *testing.T) {
|
||||
oldMem, oldGroup := gpuMemory, processGroup
|
||||
t.Cleanup(func() { gpuMemory, processGroup = oldMem, oldGroup })
|
||||
processGroup = func(pid int) int { return pid }
|
||||
gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) {
|
||||
return []discovery.GPUMemory{
|
||||
{Index: 0, UsedMB: 20000, Procs: []discovery.GPUProcess{{PID: 700, UsedMB: 2048}, {PID: 900, UsedMB: 17000}}},
|
||||
{Index: 1, UsedMB: 0},
|
||||
}, nil
|
||||
}
|
||||
a := &Agent{log: slog.Default()}
|
||||
if c, ok := a.gpuCompute(context.Background(), []int{0}, 700); !ok || c.Backend != "CUDA" ||
|
||||
len(c.Devices) != 1 || c.Devices[0] != "GPU 0: 2.0 GB" {
|
||||
t.Errorf("on GPU 0: %+v %v", c, ok)
|
||||
}
|
||||
if c, ok := a.gpuCompute(context.Background(), []int{1}, 700); !ok || c.Backend != "CPU" {
|
||||
t.Errorf("placed on GPU 1, holding none of it: %+v", c)
|
||||
}
|
||||
if c, ok := a.gpuCompute(context.Background(), nil, 700); !ok || c.Backend != "CPU" {
|
||||
t.Errorf("a CPU shard: %+v", c)
|
||||
}
|
||||
gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) { return nil, nil }
|
||||
if _, ok := a.gpuCompute(context.Background(), []int{0}, 700); ok {
|
||||
t.Error("no nvidia-smi read as an answer")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package provision
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
)
|
||||
|
||||
// comfyuiSource is a ComfyUI release archive as GitHub makes one, and the
|
||||
// recipe pointed at it.
|
||||
func comfyuiSource(t *testing.T, in *Installer, rel *release, digest string) {
|
||||
t.Helper()
|
||||
archive := tarGz(t, map[string]string{
|
||||
"ComfyUI-0.39.2/main.py": "print('comfy')\n",
|
||||
"ComfyUI-0.39.2/comfyui_version.py": "__version__ = \"0.39.2\"\n",
|
||||
"ComfyUI-0.39.2/requirements.txt": "comfyui-frontend-package==1.53.10\ntorch\ntorchvision\n" +
|
||||
"numpy>=1.25.0\n\n#non essential dependencies:\nkornia>=0.7.1\n",
|
||||
}, nil)
|
||||
rel.assets["v0.39.2.tar.gz"] = archive
|
||||
sum := sha256.Sum256(archive)
|
||||
if digest == "" {
|
||||
digest = hex.EncodeToString(sum[:])
|
||||
}
|
||||
old := comfyuiReleases
|
||||
comfyuiReleases = map[string]struct {
|
||||
sha256 string
|
||||
published string
|
||||
}{"0.39.2": {digest, "2026-10-07T21:58:07Z"}}
|
||||
t.Cleanup(func() { comfyuiReleases = old })
|
||||
}
|
||||
|
||||
var turingNode = discovery.Host{OSID: "ubuntu", OSVersion: "24.04", Arch: "amd64",
|
||||
NVIDIADriver: "580.82.07", MinComputeCapability: 7.5}
|
||||
|
||||
func TestComfyUIIsInstalledFromItsSourceCheckedAgainstTheRecipe(t *testing.T) {
|
||||
sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
|
||||
in, rel := pythonInstaller(t, turingNode, sys)
|
||||
comfyuiSource(t, in, rel, "")
|
||||
|
||||
got := in.Converge(t.Context(), []Want{{Engine: "comfyui", Version: "0.39.2"}})
|
||||
if got[0].State != Ready {
|
||||
t.Fatalf("not ready: %+v", got[0])
|
||||
}
|
||||
dir := filepath.Join(in.Root, "comfyui", "0.39.2-cu13")
|
||||
if got[0].Binary != filepath.Join(dir, "bin", "python") {
|
||||
t.Errorf("binary %s, want the environment's interpreter", got[0].Binary)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "comfyui", "main.py")); err != nil {
|
||||
t.Errorf("ComfyUI's code is not beside the interpreter: %v", err)
|
||||
}
|
||||
all := strings.Join(sys.commands, "\n")
|
||||
for _, want := range []string{
|
||||
// Not PyPI's date, which ComfyUI has none of: the recipe's, a day on.
|
||||
"--exclude-newer 2026-10-08T21:58:07Z",
|
||||
"comfyui-frontend-package==1.53.10", "numpy>=1.25.0", "kornia>=0.7.1", "torch",
|
||||
"--require-hashes",
|
||||
} {
|
||||
if !strings.Contains(all, want) {
|
||||
t.Errorf("no %q in:\n%s", want, all)
|
||||
}
|
||||
}
|
||||
if strings.Contains(all, "/pypi/") {
|
||||
t.Error("PyPI was asked when ComfyUI was published")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComfyUIOnAnOlderDriverTakesPyTorchsCUDA12Torch(t *testing.T) {
|
||||
sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
|
||||
host := turingNode
|
||||
host.NVIDIADriver = "550.163.01"
|
||||
in, rel := pythonInstaller(t, host, sys)
|
||||
comfyuiSource(t, in, rel, "")
|
||||
got := in.Converge(t.Context(), []Want{{Engine: "comfyui", Version: "0.39.2"}})
|
||||
if got[0].State != Ready || got[0].Variant != "cu129" {
|
||||
t.Fatalf("got %+v", got[0])
|
||||
}
|
||||
var project string
|
||||
for _, c := range sys.commands {
|
||||
if strings.HasPrefix(c, "pyproject: ") {
|
||||
project = c
|
||||
}
|
||||
}
|
||||
// torch from PyTorch's index alone, named once, as the build names it.
|
||||
if !strings.Contains(project, `torch = { index = "extra" }`) || strings.Count(project, `"torch"`) != 1 {
|
||||
t.Errorf("torch is not from PyTorch's index alone:\n%s", project)
|
||||
}
|
||||
}
|
||||
|
||||
// A V100 on driver 580: CUDA 13 runs on the driver, but neither its torch
|
||||
// nor CUDA 12.9's has kernels for Volta. Tried in turn, each was installed,
|
||||
// 10 GB at a time, and failed on the card.
|
||||
func TestComfyUIOnAVoltaTakesPyTorchsCUDA126Torch(t *testing.T) {
|
||||
sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
|
||||
host := turingNode
|
||||
host.MinComputeCapability = 7.0
|
||||
in, rel := pythonInstaller(t, host, sys)
|
||||
comfyuiSource(t, in, rel, "")
|
||||
got := in.Converge(t.Context(), []Want{{Engine: "comfyui", Version: "0.39.2"}})
|
||||
if got[0].State != Ready || got[0].Variant != "cu126" {
|
||||
t.Fatalf("got %+v", got[0])
|
||||
}
|
||||
all := strings.Join(sys.commands, "\n")
|
||||
if !strings.Contains(all, pytorchCU126) || strings.Contains(all, pytorchCU129) {
|
||||
t.Errorf("torch is not CUDA 12.6's:\n%s", all)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComfyUIWhoseSourceIsNotTheOneTheRecipeNamesIsNotInstalled(t *testing.T) {
|
||||
sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
|
||||
in, rel := pythonInstaller(t, turingNode, sys)
|
||||
comfyuiSource(t, in, rel, strings.Repeat("ab", 32))
|
||||
got := in.Converge(t.Context(), []Want{{Engine: "comfyui", Version: "0.39.2"}})
|
||||
if got[0].State != Failed || !strings.Contains(got[0].Detail, "does not match") {
|
||||
t.Fatalf("got %s %q", got[0].State, got[0].Detail)
|
||||
}
|
||||
for _, c := range sys.commands {
|
||||
if strings.Contains(c, "pip install") {
|
||||
t.Errorf("installed from a source that did not match: %s", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestComfyUIIsInstalledOnlyWhereItCanRun(t *testing.T) {
|
||||
sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
|
||||
in, _ := pythonInstaller(t, ubuntu, sys)
|
||||
if got := in.Converge(t.Context(), []Want{{Engine: "comfyui", Version: "0.39.2"}}); got[0].State != Skipped {
|
||||
t.Errorf("a node with no GPU: %+v", got[0])
|
||||
}
|
||||
in, _ = pythonInstaller(t, turingNode, sys)
|
||||
got := in.Converge(t.Context(), []Want{{Engine: "comfyui", Version: "0.1.0"}})
|
||||
if got[0].State != Unsupported || !strings.Contains(got[0].Detail, "knows the source digest of ComfyUI 0.39.2") {
|
||||
t.Errorf("a version without a digest: %s %q", got[0].State, got[0].Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARequirementsFileIsReadForPackagesAndNothingElse(t *testing.T) {
|
||||
got, err := requirementsIn("torch\ntorchvision>=0.20\nPillow\n# a comment\naiohttp>=3.11 # inline\n", pytorchPackages)
|
||||
if err != nil || strings.Join(got, ",") != "Pillow,aiohttp>=3.11" {
|
||||
t.Errorf("%v %v", got, err)
|
||||
}
|
||||
for _, bad := range []string{"-r other.txt", "--index-url https://evil", "x @ https://evil/x.whl", "-e ."} {
|
||||
if _, err := requirementsIn(bad+"\n", nil); err == nil {
|
||||
t.Errorf("%q was accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A card every build runs on keeps the order it had: a Turing node's
|
||||
// installed cu13 is still the first found, and is not installed again
|
||||
// because a build for older cards was added after it.
|
||||
func TestComfyUIBuildsAreTriedBestFirst(t *testing.T) {
|
||||
variants := func(h discovery.Host) string {
|
||||
builds, err := comfyuiCandidates("0.39.2", "", h)
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
var v []string
|
||||
for _, b := range builds {
|
||||
v = append(v, b.Variant)
|
||||
}
|
||||
return strings.Join(v, ",")
|
||||
}
|
||||
volta := turingNode
|
||||
volta.MinComputeCapability = 7.0
|
||||
older := turingNode
|
||||
older.NVIDIADriver = "550.163.01"
|
||||
for h, want := range map[*discovery.Host]string{&turingNode: "cu13,cu129,cu126", &volta: "cu126", &older: "cu129,cu126"} {
|
||||
if got := variants(*h); got != want {
|
||||
t.Errorf("cc %.1f driver %s: %s, want %s", h.MinComputeCapability, h.NVIDIADriver, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -66,12 +66,17 @@ func (in *Installer) installPython(ctx context.Context, r Recipe, w Want, b Buil
|
||||
return fmt.Errorf("installing uv, which installs %s: %s", w.Engine, uv.Detail)
|
||||
}
|
||||
|
||||
in.progress(st, "asking PyPI when %s %s was published", r.PyPI, w.Version)
|
||||
published, err := in.published(ctx, r.PyPI, w.Version)
|
||||
if err != nil {
|
||||
return err
|
||||
var asOf string
|
||||
if !b.AsOf.IsZero() {
|
||||
asOf = b.AsOf.UTC().Format(time.RFC3339)
|
||||
} else {
|
||||
in.progress(st, "asking PyPI when %s %s was published", r.PyPI, w.Version)
|
||||
published, err := in.published(ctx, r.PyPI, w.Version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
asOf = published.Add(24 * time.Hour).UTC().Format(time.RFC3339)
|
||||
}
|
||||
asOf := published.Add(24 * time.Hour).UTC().Format(time.RFC3339)
|
||||
|
||||
cache := filepath.Join(in.root(), ".uv-cache")
|
||||
env := []string{
|
||||
@@ -96,7 +101,14 @@ func (in *Installer) installPython(ctx context.Context, r Recipe, w Want, b Buil
|
||||
}
|
||||
python := filepath.Join(tmp, "bin", "python")
|
||||
|
||||
reqs := b.Requirements
|
||||
reqs := slices.Clone(b.Requirements)
|
||||
if b.Source != nil {
|
||||
more, err := in.unpackSource(ctx, w, b, tmp, st)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reqs = append(reqs, more...)
|
||||
}
|
||||
if b.Wheel != nil {
|
||||
// The engine from its GitHub release, pinned to the digest GitHub
|
||||
// publishes for the file: the same trust as an archive, and uv
|
||||
@@ -320,3 +332,68 @@ func (in *Installer) gpuCheck(ctx context.Context, python string, w Want, b Buil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unpackSource fetches an engine's source archive, checks it against the
|
||||
// digest the recipe holds, puts its code in the environment, and returns the
|
||||
// requirements its requirements file names — less the packages a second
|
||||
// index gives, which the build names itself.
|
||||
func (in *Installer) unpackSource(ctx context.Context, w Want, b Build, env string, st *Status) ([]string, error) {
|
||||
src := b.Source
|
||||
url := src.URL
|
||||
if strings.HasPrefix(url, "/") {
|
||||
url = strings.TrimSuffix(in.downloadBase(), "/") + url
|
||||
}
|
||||
scratch := env + ".source"
|
||||
_ = os.RemoveAll(scratch)
|
||||
if err := os.MkdirAll(scratch, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(scratch) //nolint:errcheck // scratch
|
||||
archive := filepath.Join(scratch, "source.tar.gz")
|
||||
name := w.Engine + " " + w.Version + "'s source"
|
||||
if err := in.download(ctx, url, name, archive, src.SHA256, st); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
in.progress(st, "unpacking %s", name)
|
||||
if err := untarGz(archive, scratch); err != nil {
|
||||
return nil, fmt.Errorf("unpacking %s: %w", name, err)
|
||||
}
|
||||
dst := filepath.Join(env, src.Dir)
|
||||
if err := os.Rename(filepath.Join(scratch, src.Top), dst); err != nil {
|
||||
return nil, fmt.Errorf("%s has no %s at its top: %w", name, src.Top, err)
|
||||
}
|
||||
if src.Requirements == "" {
|
||||
return nil, nil
|
||||
}
|
||||
f, err := os.ReadFile(filepath.Join(dst, src.Requirements))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", name, err)
|
||||
}
|
||||
return requirementsIn(string(f), b.IndexPackages)
|
||||
}
|
||||
|
||||
// requirementsIn reads a requirements file's lines. Options (-r, -e, --index-url
|
||||
// and the like) are refused: they would fetch from somewhere other than PyPI,
|
||||
// or install what the lock never saw.
|
||||
func requirementsIn(text string, skip []string) ([]string, error) {
|
||||
var out []string
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
line, _, _ = strings.Cut(line, "#")
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(line, "-") || strings.Contains(line, "://") || strings.Contains(line, " @ ") {
|
||||
return nil, fmt.Errorf("the requirements name %q, which is not a package from PyPI", line)
|
||||
}
|
||||
name := strings.ToLower(line)
|
||||
if i := strings.IndexAny(name, "<>=!~;[ "); i >= 0 {
|
||||
name = name[:i]
|
||||
}
|
||||
if slices.Contains(skip, name) {
|
||||
continue
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -53,6 +53,7 @@ func (f *fakePython) run(ctx context.Context, env []string, name string, args ..
|
||||
return []byte("error: Hash mismatch for torch\n"), errExit
|
||||
}
|
||||
venv := filepath.Dir(filepath.Dir(args[slices.Index(args, "--python")+1]))
|
||||
_ = os.WriteFile(filepath.Join(venv, "bin", "python"), []byte("#!/bin/sh\n"), 0o755)
|
||||
return nil, os.WriteFile(filepath.Join(venv, "bin", "vllm"), []byte("#!/bin/sh\n"), 0o755)
|
||||
case strings.HasSuffix(name, "/bin/vllm") && args[0] == "--version":
|
||||
return []byte("0.30.0\n"), nil
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
)
|
||||
@@ -68,12 +69,31 @@ type Build struct {
|
||||
// Note says why this build, when it is not the plain one: shown beside
|
||||
// it wherever the installation is.
|
||||
Note string
|
||||
// Source, for a Python engine that is not on PyPI, is its own code: an
|
||||
// archive pinned by its digest, unpacked into the environment, whose
|
||||
// requirements file is installed with Requirements. AsOf is then the
|
||||
// date its dependencies are resolved as of — the day after its version
|
||||
// was published, which PyPI cannot be asked for.
|
||||
Source *Source
|
||||
AsOf time.Time
|
||||
// FreeGB is the disk space the installation needs before it starts: a
|
||||
// Python engine is several gigabytes, and one that fills the disk halfway
|
||||
// leaves the node worse off than one that never began.
|
||||
FreeGB int
|
||||
}
|
||||
|
||||
// Source is an engine's code as an archive.
|
||||
type Source struct {
|
||||
// URL is fetched as is; SHA256 is what it must hash to, written into
|
||||
// the recipe because a project's source archives come with no published
|
||||
// digest of their own.
|
||||
URL, SHA256 string
|
||||
// Top is the archive's one top directory, put in the environment as Dir.
|
||||
Top, Dir string
|
||||
// Requirements is the requirements file, in Dir.
|
||||
Requirements string
|
||||
}
|
||||
|
||||
// Wheel is a package published as a GitHub release asset: the asset whose
|
||||
// name starts with Prefix and ends with Suffix.
|
||||
type Wheel struct {
|
||||
@@ -104,6 +124,7 @@ var Recipes = map[string]Recipe{
|
||||
"llamacpp": {Repo: "ggml-org/llama.cpp", Candidates: llamaCppCandidates},
|
||||
"vllm": {PyPI: "vllm", Candidates: vllmCandidates},
|
||||
"sglang": {PyPI: "sglang", Candidates: sglangCandidates},
|
||||
"comfyui": {Candidates: comfyuiCandidates},
|
||||
"uv": {Repo: "astral-sh/uv", Candidates: uvCandidates},
|
||||
}
|
||||
|
||||
@@ -556,3 +577,98 @@ func orUnknown(s string) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// --- ComfyUI ---
|
||||
|
||||
// comfyuiReleases are the ComfyUI versions this agent installs: each source
|
||||
// archive's digest, taken when it was added here and confirmed from a second
|
||||
// machine, and when the version was published. GitHub publishes no digest
|
||||
// for a tag's archive, so a version not listed is refused rather than
|
||||
// installed unchecked.
|
||||
var comfyuiReleases = map[string]struct {
|
||||
sha256 string
|
||||
published string
|
||||
}{
|
||||
"0.39.2": {"1d655e22cf546700fc1d6d2536e27701695c5832a51f63bb16c41866ac09be0b", "2026-10-07T21:58:07Z"},
|
||||
}
|
||||
|
||||
// ComfyUIVersion is the newest ComfyUI this agent installs: what a cluster
|
||||
// is set to when a model first needs it.
|
||||
func ComfyUIVersion() string {
|
||||
newest := ""
|
||||
for v := range comfyuiReleases {
|
||||
if newest == "" || versionBefore(newest, v) {
|
||||
newest = v
|
||||
}
|
||||
}
|
||||
return newest
|
||||
}
|
||||
|
||||
// ComfyUIRepo is where ComfyUI's source is published.
|
||||
const ComfyUIRepo = "Comfy-Org/ComfyUI"
|
||||
|
||||
// comfyuiDir is where ComfyUI's code is put in its environment: main.py is
|
||||
// <environment>/comfyui/main.py, beside bin/python.
|
||||
const comfyuiDir = "comfyui"
|
||||
|
||||
// pytorchCU126 is PyTorch's CUDA 12.6 build: the last that carries kernels
|
||||
// for Volta and Pascal. From torch 2.8 its CUDA 12.8 and 12.9 builds start
|
||||
// at Turing (7.5), as CUDA 13 does, so a V100 has no torch but this one.
|
||||
const pytorchCU126 = "https://download.pytorch.org/whl/cu126"
|
||||
|
||||
// comfyuiCandidates are the ComfyUI builds for a machine: its source, in a
|
||||
// Python environment with the torch for the node's driver and cards —
|
||||
// PyPI's, built for CUDA 13; PyTorch's CUDA 12.9 build for a driver CUDA 13
|
||||
// does not run on; and its CUDA 12.6 build for cards older than Turing.
|
||||
func comfyuiCandidates(version, variant string, h discovery.Host) ([]Build, error) {
|
||||
if _, err := gpuArch("ComfyUI", h); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rel, ok := comfyuiReleases[version]
|
||||
if !ok {
|
||||
known := make([]string, 0, len(comfyuiReleases))
|
||||
for v := range comfyuiReleases {
|
||||
known = append(known, v)
|
||||
}
|
||||
slices.Sort(known)
|
||||
return nil, fmt.Errorf("this agent knows the source digest of ComfyUI %s, not of %s; "+
|
||||
"install that one by hand and name it in agent.yaml (engines.comfyui.dir)",
|
||||
strings.Join(known, ", "), version)
|
||||
}
|
||||
published, err := time.Parse(time.RFC3339, rel.published)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
build := func(variant string) Build {
|
||||
return Build{
|
||||
Variant: variant,
|
||||
Python: pythonForEngines,
|
||||
Source: &Source{
|
||||
URL: "/" + ComfyUIRepo + "/archive/refs/tags/v" + version + ".tar.gz",
|
||||
SHA256: rel.sha256, Top: "ComfyUI-" + version, Dir: comfyuiDir,
|
||||
Requirements: "requirements.txt",
|
||||
},
|
||||
AsOf: published.Add(24 * time.Hour),
|
||||
// ComfyUI is started as a script, so what the agent is given is
|
||||
// the environment's interpreter; the proof is importing what it
|
||||
// imports first, from where its code is.
|
||||
Binary: "python",
|
||||
VersionArgs: []string{"-c", `import os, sys, torch, aiohttp, safetensors
|
||||
d = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(sys.executable))), "` + comfyuiDir + `")
|
||||
assert os.path.isfile(os.path.join(d, "main.py")), d + " has no main.py"
|
||||
print("comfyui version", "` + version + `", "on torch", torch.__version__)`},
|
||||
Packages: compilerPackages,
|
||||
FreeGB: 15,
|
||||
GPUCheck: true,
|
||||
}
|
||||
}
|
||||
cu129 := build("cu129")
|
||||
cu129.ExtraIndex, cu129.IndexPackages = pytorchCU129, pytorchPackages
|
||||
cu126 := build("cu126")
|
||||
cu126.ExtraIndex, cu126.IndexPackages = pytorchCU126, pytorchPackages
|
||||
return pick("ComfyUI "+version, variant, []candidate{
|
||||
{build("cu13"), gpuNeeds{minDriverForCUDA13, 7.5}, "ComfyUI " + version + " on CUDA 13's torch"},
|
||||
{cu129, gpuNeeds{minDriverForCUDA12, 7.5}, "ComfyUI " + version + " on CUDA 12.9's torch"},
|
||||
{cu126, gpuNeeds{minDriverForCUDA12, 5.0}, "ComfyUI " + version + " on CUDA 12.6's torch"},
|
||||
}, h)
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
)
|
||||
|
||||
// The cluster's settings for this agent (BACKLOG E3): what agent.yaml can
|
||||
// say but the agent's id, control planes, address and certificate, kept by
|
||||
// the control planes and sent over the stream. Kept here beside agent.yaml,
|
||||
// which then holds only what the agent needs to start; at start the two are
|
||||
// read together, these over the file.
|
||||
|
||||
// SettingsFile keeps the settings the control plane last sent.
|
||||
var SettingsFile = "/var/lib/llmgw-agent/agent-settings.yaml"
|
||||
|
||||
// ReadSettings is what SettingsFile holds, nil when nothing.
|
||||
func ReadSettings() []byte {
|
||||
b, err := os.ReadFile(SettingsFile)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// SetSettingsCheck says how to tell whether settings would load beside this
|
||||
// agent's own file; settings that would not are not taken.
|
||||
func (a *Agent) SetSettingsCheck(f func([]byte) error) { a.settingsCheck = f }
|
||||
|
||||
// applySettings takes the settings the control plane sent: nothing when they
|
||||
// are what this agent runs on, and otherwise kept and started again on.
|
||||
func (a *Agent) applySettings(m *llmgwv1.AgentSettings) {
|
||||
next := m.GetYaml()
|
||||
if next == a.cfg.Settings {
|
||||
return
|
||||
}
|
||||
if a.settingsCheck != nil {
|
||||
if err := a.settingsCheck([]byte(next)); err != nil {
|
||||
// Kept running on what it has: settings that will not load
|
||||
// here must not take the node out of service.
|
||||
a.log.Error("the cluster's settings for this agent do not load beside its agent.yaml; running on what it has",
|
||||
"err", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(SettingsFile), 0o755); err != nil {
|
||||
a.log.Error("could not keep the cluster's settings for this agent", "err", err)
|
||||
return
|
||||
}
|
||||
tmp := SettingsFile + ".new"
|
||||
if err := os.WriteFile(tmp, []byte(next), 0o600); err != nil {
|
||||
a.log.Error("could not keep the cluster's settings for this agent", "err", err)
|
||||
return
|
||||
}
|
||||
if err := os.Rename(tmp, SettingsFile); err != nil {
|
||||
a.log.Error("could not keep the cluster's settings for this agent", "err", err)
|
||||
return
|
||||
}
|
||||
a.log.Info("the cluster's settings for this agent changed; starting again on them, the engines keep serving")
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
exitForUpgrade()
|
||||
}
|
||||
|
||||
// WatchFile starts the agent again when its configuration file changes and
|
||||
// still loads: agent.yaml is read only at start, and a change to it that
|
||||
// waited for somebody to restart the agent was a change nobody saw take
|
||||
// effect.
|
||||
func (a *Agent) WatchFile(ctx context.Context, path string, loads func() error) {
|
||||
if path == "" {
|
||||
return
|
||||
}
|
||||
sum := func() [32]byte {
|
||||
b, _ := os.ReadFile(path) //nolint:gosec // the agent's own configuration
|
||||
return sha256.Sum256(b)
|
||||
}
|
||||
was := sum()
|
||||
t := time.NewTicker(watchEvery)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
now := sum()
|
||||
if now == was {
|
||||
continue
|
||||
}
|
||||
if err := loads(); err != nil {
|
||||
// Once per change: an editor saving halfway must not stop the
|
||||
// agent, and a broken file is reported, not restarted into.
|
||||
a.log.Error("agent.yaml changed and does not load; running on what it had", "path", path, "err", err)
|
||||
was = now
|
||||
continue
|
||||
}
|
||||
a.log.Info("agent.yaml changed; starting again on it, the engines keep serving", "path", path)
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
exitForUpgrade()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// watchEvery is how often agent.yaml is looked at.
|
||||
var watchEvery = 10 * time.Second
|
||||
@@ -0,0 +1,88 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
)
|
||||
|
||||
func settingsAgent(t *testing.T, running string) (*Agent, chan struct{}) {
|
||||
t.Helper()
|
||||
oldFile, oldExit := SettingsFile, exitForUpgrade
|
||||
SettingsFile = filepath.Join(t.TempDir(), "agent-settings.yaml")
|
||||
exited := make(chan struct{}, 1)
|
||||
exitForUpgrade = func() { exited <- struct{}{} }
|
||||
t.Cleanup(func() { SettingsFile, exitForUpgrade = oldFile, oldExit })
|
||||
cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(), EnginesDir: t.TempDir(), Settings: running}
|
||||
cfg.Superv.WorkDir = t.TempDir()
|
||||
a, err := New(cfg, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return a, exited
|
||||
}
|
||||
|
||||
// exitedSoon is whether the agent exited within a second: it waits a
|
||||
// moment for its log line to leave first.
|
||||
func exitedSoon(ch chan struct{}) bool {
|
||||
select {
|
||||
case <-ch:
|
||||
return true
|
||||
case <-time.After(time.Second):
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAgentOnItsSettingsStaysAndOneOffThemStartsAgain(t *testing.T) {
|
||||
a, exited := settingsAgent(t, "heartbeat_interval: 5s\n")
|
||||
a.applySettings(&llmgwv1.AgentSettings{Yaml: "heartbeat_interval: 5s\n"})
|
||||
if exitedSoon(exited) || ReadSettings() != nil {
|
||||
t.Error("settings it already runs on restarted it")
|
||||
}
|
||||
a.applySettings(&llmgwv1.AgentSettings{Yaml: "heartbeat_interval: 2s\n"})
|
||||
if !exitedSoon(exited) || string(ReadSettings()) != "heartbeat_interval: 2s\n" {
|
||||
t.Errorf("new settings: kept %q", ReadSettings())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSettingsThatWouldNotLoadAreNotTaken(t *testing.T) {
|
||||
a, exited := settingsAgent(t, "")
|
||||
a.SetSettingsCheck(func([]byte) error { return errors.New("fabric ib-a lists no HCAs") })
|
||||
a.applySettings(&llmgwv1.AgentSettings{Yaml: "fabrics: [{name: ib-a}]\n"})
|
||||
if exitedSoon(exited) || ReadSettings() != nil {
|
||||
t.Error("settings that do not load were taken")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAChangedAgentYamlThatLoadsStartsTheAgentAgain(t *testing.T) {
|
||||
a, exited := settingsAgent(t, "")
|
||||
old := watchEvery
|
||||
watchEvery = 10 * time.Millisecond
|
||||
t.Cleanup(func() { watchEvery = old })
|
||||
p := filepath.Join(t.TempDir(), "agent.yaml")
|
||||
_ = os.WriteFile(p, []byte("node_id: n1\n"), 0o600)
|
||||
var loads atomic.Bool
|
||||
check := func() error {
|
||||
if loads.Load() {
|
||||
return nil
|
||||
}
|
||||
return errors.New("broken")
|
||||
}
|
||||
go a.WatchFile(t.Context(), p, check)
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
// Saved broken: reported, not restarted into.
|
||||
_ = os.WriteFile(p, []byte("node_id: [\n"), 0o600)
|
||||
if exitedSoon(exited) {
|
||||
t.Fatal("a broken agent.yaml restarted the agent")
|
||||
}
|
||||
loads.Store(true)
|
||||
_ = os.WriteFile(p, []byte("node_id: n1\nlabels: {a: b}\n"), 0o600)
|
||||
if !exitedSoon(exited) {
|
||||
t.Error("a changed agent.yaml did not restart the agent")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package supervisor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// An engine's output goes to a file of its own, not to a pipe of the
|
||||
// agent's. A pipe has one reader, and that reader is the agent that
|
||||
// started the engine: when that agent restarts, the engine it left running
|
||||
// — and adopts a moment later — writes to a pipe nobody reads, and its next
|
||||
// write past the pipe's buffer fails with EPIPE. A Python engine's progress
|
||||
// bar then raises, and takes the request it was reporting on with it. A
|
||||
// file has no reader to lose: the engine appends to it, every agent reads
|
||||
// it, and the output from before a restart is there for the one after.
|
||||
|
||||
// maxOutputBytes bounds one process's file: past it the file is emptied and
|
||||
// the process goes on appending to it. A progress bar rewriting itself for
|
||||
// an hour would otherwise fill the disk.
|
||||
const maxOutputBytes = 64 << 20
|
||||
|
||||
// outputPath is where a shard's process writes.
|
||||
func (s *Supervisor) outputPath(shardID, name string) string {
|
||||
if name != "" {
|
||||
return filepath.Join(s.cfg.WorkDir, "shard-"+shardID+"."+name+".log")
|
||||
}
|
||||
return filepath.Join(s.cfg.WorkDir, "shard-"+shardID+".log")
|
||||
}
|
||||
|
||||
// openOutput is the file a process is started with: new, for a process
|
||||
// that is new.
|
||||
func (s *Supervisor) openOutput(shardID, name string) (*os.File, error) {
|
||||
return os.OpenFile(s.outputPath(shardID, name), os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_TRUNC, 0o600) //nolint:gosec // the agent's own directory
|
||||
}
|
||||
|
||||
// follow reads a process's file into the shard's log as it is written,
|
||||
// until the shard's output is stopped and the file has been read to its
|
||||
// end. A file an earlier agent began is read from the start, so the output
|
||||
// from before a restart is in the log too.
|
||||
func (s *Supervisor) follow(sh *Shard, path, name string) {
|
||||
defer sh.outDone.Done()
|
||||
f, err := os.Open(path) //nolint:gosec // the agent's own directory
|
||||
if err != nil {
|
||||
sh.logs.add("[" + name + "] output unavailable: " + err.Error())
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
var offset int64
|
||||
var rest []byte
|
||||
buf := make([]byte, 64<<10)
|
||||
read := func() {
|
||||
for {
|
||||
st, err := f.Stat()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if st.Size() < offset {
|
||||
// Emptied, by this agent or another: begin again.
|
||||
offset, rest = 0, nil
|
||||
}
|
||||
if st.Size() > maxOutputBytes {
|
||||
_ = os.Truncate(path, 0)
|
||||
offset, rest = 0, nil
|
||||
continue
|
||||
}
|
||||
n, err := f.ReadAt(buf, offset)
|
||||
if n > 0 {
|
||||
offset += int64(n)
|
||||
rest = append(rest, buf[:n]...)
|
||||
for {
|
||||
i := bytes.IndexByte(rest, '\n')
|
||||
if i < 0 {
|
||||
break
|
||||
}
|
||||
s.addLine(sh, name, rest[:i])
|
||||
rest = rest[i+1:]
|
||||
}
|
||||
// A progress bar is a line rewritten with \r and never ended:
|
||||
// its latest state is what is wanted, not a line a megabyte
|
||||
// long. Kept, as llama.cpp and tqdm write them.
|
||||
if len(rest) > 1<<20 {
|
||||
s.addLine(sh, name, rest)
|
||||
rest = nil
|
||||
}
|
||||
}
|
||||
if err != nil || n == 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
t := time.NewTicker(200 * time.Millisecond)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-sh.outStop:
|
||||
read()
|
||||
if len(rest) > 0 {
|
||||
s.addLine(sh, name, rest)
|
||||
}
|
||||
return
|
||||
case <-t.C:
|
||||
read()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Supervisor) addLine(sh *Shard, name string, b []byte) {
|
||||
// The last state of a line rewritten with \r, as a terminal would show.
|
||||
if i := bytes.LastIndexByte(b, '\r'); i >= 0 {
|
||||
b = b[i+1:]
|
||||
}
|
||||
line := string(b)
|
||||
if name != "" {
|
||||
// Prefixed so the two processes of a Ray shard stay distinguishable
|
||||
// in a single log tail.
|
||||
line = "[" + name + "] " + line
|
||||
}
|
||||
sh.logs.add(line)
|
||||
s.log.Debug("engine", "shard", sh.ID, "line", line)
|
||||
}
|
||||
|
||||
// removeOutput drops a shard's files once nothing writes to them.
|
||||
func (s *Supervisor) removeOutput(shardID string) {
|
||||
matches, _ := filepath.Glob(filepath.Join(s.cfg.WorkDir, "shard-"+shardID+".*log"))
|
||||
for _, m := range matches {
|
||||
_ = os.Remove(m)
|
||||
}
|
||||
_ = os.Remove(s.outputPath(shardID, ""))
|
||||
}
|
||||
|
||||
// stopOutput has every follower read what is left and stop. Called once
|
||||
// the process has ended, before the shard is done, so what it wrote last
|
||||
// is in the log by the time anyone reads the log for why it ended.
|
||||
func (s *Supervisor) stopOutput(sh *Shard) {
|
||||
sh.outOnce.Do(func() { close(sh.outStop) })
|
||||
sh.outDone.Wait()
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package supervisor
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
@@ -90,3 +91,25 @@ func credentialFor(u *user.User, workDir string) (*runAs, error) {
|
||||
func (r *runAs) env() []string {
|
||||
return []string{"HOME=" + r.home, "XDG_CACHE_HOME=" + filepath.Join(r.home, ".cache"), "USER=" + r.name, "LOGNAME=" + r.name}
|
||||
}
|
||||
|
||||
// AsEngineUser has cmd run as the engine user, as an engine would, with its
|
||||
// home under workDir: for work the agent does that is not an engine but
|
||||
// reads what an engine reads — training an adapter. It answers that home
|
||||
// and a way to hand a path to the user, or why it runs as this process.
|
||||
func AsEngineUser(cmd *exec.Cmd, name, workDir string) (home string, give func(path string) error, why string) {
|
||||
r, why := resolveRunAs(name, workDir)
|
||||
if r == nil {
|
||||
return "", func(string) error { return nil }, why
|
||||
}
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Credential: r.cred, Setpgid: true}
|
||||
cmd.Env = append(cmd.Env, r.env()...)
|
||||
uid, gid := int(r.cred.Uid), int(r.cred.Gid)
|
||||
return r.home, func(path string) error {
|
||||
return filepath.WalkDir(path, func(p string, _ os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Lchown(p, uid, gid)
|
||||
})
|
||||
}, ""
|
||||
}
|
||||
@@ -1,12 +1,10 @@
|
||||
package supervisor
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
@@ -104,6 +102,12 @@ type Shard struct {
|
||||
ID string
|
||||
Spec engines.ShardSpec
|
||||
|
||||
// outStop ends the followers of the processes' output files, and
|
||||
// outDone is their having read to the end.
|
||||
outStop chan struct{}
|
||||
outOnce sync.Once
|
||||
outDone sync.WaitGroup
|
||||
|
||||
mu sync.Mutex
|
||||
state State
|
||||
pid int
|
||||
@@ -113,7 +117,10 @@ type Shard struct {
|
||||
exitCode int
|
||||
|
||||
logs *ringBuffer
|
||||
cmd *exec.Cmd
|
||||
// cmdline is an adopted engine's command line, from its state file: it
|
||||
// has no cmd of this agent's to read it from.
|
||||
cmdline []string
|
||||
cmd *exec.Cmd
|
||||
// sidecars are the shard's supporting processes, torn down with it.
|
||||
sidecars []*exec.Cmd
|
||||
cancel context.CancelFunc
|
||||
@@ -245,6 +252,7 @@ func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpe
|
||||
logs: newRingBuffer(s.cfg.LogLines),
|
||||
started: time.Now(),
|
||||
done: make(chan struct{}),
|
||||
outStop: make(chan struct{}),
|
||||
}
|
||||
s.shards[id] = sh
|
||||
s.mu.Unlock()
|
||||
@@ -312,14 +320,15 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
|
||||
// CommandContext kills only the leader, so replace that with a group kill.
|
||||
c.Cancel = func() error { return killGroup(c.Process.Pid, syscall.SIGKILL) }
|
||||
|
||||
stdout, err := c.StdoutPipe()
|
||||
out, err := s.openOutput(sh.ID, "")
|
||||
if err != nil {
|
||||
cancel()
|
||||
return err
|
||||
}
|
||||
c.Stderr = c.Stdout.(io.Writer) // interleave, as the engine writes them
|
||||
|
||||
if err := c.Start(); err != nil {
|
||||
c.Stdout, c.Stderr = out, out // interleaved, as the engine writes them
|
||||
err = c.Start()
|
||||
out.Close() // the process holds its own
|
||||
if err != nil {
|
||||
cancel()
|
||||
return fmt.Errorf("starting %s: %w", cmd.Path, err)
|
||||
}
|
||||
@@ -336,7 +345,8 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
|
||||
"shard", sh.ID, "err", err)
|
||||
}
|
||||
|
||||
go s.pump(sh, stdout, cmd.Name)
|
||||
sh.outDone.Add(1)
|
||||
go s.follow(sh, s.outputPath(sh.ID, ""), cmd.Name)
|
||||
go s.wait(sh)
|
||||
return nil
|
||||
}
|
||||
@@ -350,16 +360,18 @@ func (s *Supervisor) spawn(ctx context.Context, sh *Shard, c engines.Command) (*
|
||||
s.dropTo(proc)
|
||||
proc.Cancel = func() error { return killGroup(proc.Process.Pid, syscall.SIGKILL) }
|
||||
|
||||
stdout, err := proc.StdoutPipe()
|
||||
out, err := s.openOutput(sh.ID, c.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
proc.Stderr = proc.Stdout.(io.Writer)
|
||||
|
||||
if err := proc.Start(); err != nil {
|
||||
proc.Stdout, proc.Stderr = out, out
|
||||
err = proc.Start()
|
||||
out.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("starting %s: %w", c.Path, err)
|
||||
}
|
||||
go s.pump(sh, stdout, c.Name)
|
||||
sh.outDone.Add(1)
|
||||
go s.follow(sh, s.outputPath(sh.ID, c.Name), c.Name)
|
||||
// Reaped so it does not become a zombie; its exit does not end the shard.
|
||||
go func() { _ = proc.Wait() }()
|
||||
return proc, nil
|
||||
@@ -392,24 +404,6 @@ func waitForPort(ctx context.Context, addr string, timeout time.Duration) (err e
|
||||
}
|
||||
|
||||
// pump copies a process's output into the ring buffer and the log.
|
||||
func (s *Supervisor) pump(sh *Shard, r io.Reader, name string) {
|
||||
sc := bufio.NewScanner(r)
|
||||
// Engine tracebacks and progress bars produce long lines; the default
|
||||
// 64 KiB limit truncates them exactly when they matter.
|
||||
sc.Buffer(make([]byte, 0, 64<<10), 1<<20)
|
||||
|
||||
for sc.Scan() {
|
||||
line := sc.Text()
|
||||
if name != "" {
|
||||
// Prefixed so the two processes of a Ray shard stay distinguishable
|
||||
// in a single log tail.
|
||||
line = "[" + name + "] " + line
|
||||
}
|
||||
sh.logs.add(line)
|
||||
s.log.Debug("engine", "shard", sh.ID, "line", line)
|
||||
}
|
||||
}
|
||||
|
||||
// wait reaps the process and records why it ended.
|
||||
func (s *Supervisor) wait(sh *Shard) {
|
||||
defer close(sh.done)
|
||||
@@ -437,6 +431,7 @@ func (s *Supervisor) wait(sh *Shard) {
|
||||
final, code := sh.state, sh.exitCode
|
||||
sh.mu.Unlock()
|
||||
|
||||
s.stopOutput(sh)
|
||||
s.removeState(sh)
|
||||
if final == Failed {
|
||||
s.log.Error("shard failed", "shard", sh.ID, "exit_code", code, "err", err,
|
||||
@@ -644,6 +639,32 @@ func (s *Supervisor) Get(id string) (*Shard, error) {
|
||||
return sh, nil
|
||||
}
|
||||
|
||||
// SetCompute records where a shard's engine said it put the model, with the
|
||||
// shard's state on disk.
|
||||
func (s *Supervisor) SetCompute(id string, c engines.Compute) {
|
||||
s.mu.Lock()
|
||||
sh, ok := s.shards[id]
|
||||
if ok {
|
||||
sh.Spec.Compute = &c
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if ok && sh.pid > 0 {
|
||||
_ = s.writeState(sh)
|
||||
}
|
||||
}
|
||||
|
||||
// ComputeOf is where a shard's engine was found to compute, nil when not
|
||||
// yet known.
|
||||
func (s *Supervisor) ComputeOf(id string) *engines.Compute {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if sh, ok := s.shards[id]; ok && sh.Spec.Compute != nil {
|
||||
c := *sh.Spec.Compute
|
||||
return &c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove forgets a terminated shard.
|
||||
func (s *Supervisor) Remove(id string) {
|
||||
s.mu.Lock()
|
||||
@@ -703,11 +724,17 @@ type shardState struct {
|
||||
|
||||
// writeState records everything needed to recognise and describe the process later.
|
||||
func (s *Supervisor) writeState(sh *Shard) error {
|
||||
// An adopted engine was started by an earlier agent: its command line
|
||||
// is the one its state file kept.
|
||||
cmdline := sh.cmdline
|
||||
if sh.cmd != nil {
|
||||
cmdline = sh.cmd.Args
|
||||
}
|
||||
b, err := json.Marshal(shardState{
|
||||
ShardID: sh.ID,
|
||||
ReplicaID: sh.Spec.ReplicaID,
|
||||
PID: sh.pid,
|
||||
Cmdline: sh.cmd.Args,
|
||||
Cmdline: cmdline,
|
||||
StartedAt: sh.started,
|
||||
Spec: sh.Spec,
|
||||
})
|
||||
@@ -725,6 +752,7 @@ func (s *Supervisor) writeState(sh *Shard) error {
|
||||
|
||||
func (s *Supervisor) removeState(sh *Shard) {
|
||||
_ = os.Remove(s.statePath(sh.ID))
|
||||
s.removeOutput(sh.ID)
|
||||
}
|
||||
|
||||
// Orphan is an engine process left behind by a previous agent.
|
||||
@@ -803,14 +831,18 @@ func (s *Supervisor) Adopt(o Orphan) (*Shard, error) {
|
||||
logs: newRingBuffer(s.cfg.LogLines),
|
||||
started: o.StartedAt,
|
||||
done: make(chan struct{}),
|
||||
outStop: make(chan struct{}),
|
||||
adopted: true,
|
||||
cmdline: o.Cmdline,
|
||||
}
|
||||
s.shards[o.ShardID] = sh
|
||||
s.mu.Unlock()
|
||||
|
||||
// Output is gone with the previous agent; say so rather than presenting an
|
||||
// empty log as if the engine had been silent.
|
||||
sh.logs.add("[adopted from a previous agent; earlier output is not available]")
|
||||
// Its output file is read from the start: what it wrote under the
|
||||
// previous agent is there, and what it writes now goes on arriving.
|
||||
sh.logs.add("[adopted from a previous agent]")
|
||||
sh.outDone.Add(1)
|
||||
go s.follow(sh, s.outputPath(sh.ID, ""), "")
|
||||
|
||||
go s.poll(sh)
|
||||
return sh, nil
|
||||
@@ -838,6 +870,7 @@ func (s *Supervisor) poll(sh *Shard) {
|
||||
}
|
||||
sh.mu.Unlock()
|
||||
|
||||
s.stopOutput(sh)
|
||||
s.removeState(sh)
|
||||
s.log.Info("adopted shard exited", "shard", sh.ID, "expected", expected)
|
||||
return
|
||||
@@ -864,6 +897,7 @@ func (s *Supervisor) KillOrphan(o Orphan) error {
|
||||
}
|
||||
// An already-missing state file is success: the previous agent may have
|
||||
// reaped the process itself between the scan and this call.
|
||||
s.removeOutput(o.ShardID)
|
||||
if err := os.Remove(s.statePath(o.ShardID)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package supervisor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -848,3 +849,135 @@ func eventually(t *testing.T, limit time.Duration, msg string, cond func() bool)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatAShardComputesOnSurvivesTheAgent(t *testing.T) {
|
||||
s, _ := newSupervisor(t)
|
||||
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20100}
|
||||
sh, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Stop(context.Background(), "a") })
|
||||
_ = sh
|
||||
s.SetCompute("a", engines.Compute{Backend: "CUDA", Offloaded: 49, Total: 49})
|
||||
|
||||
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
|
||||
&scriptedHealth{}, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
orphans, err := s2.FindOrphans()
|
||||
if err != nil || len(orphans) != 1 {
|
||||
t.Fatalf("%v %d", err, len(orphans))
|
||||
}
|
||||
if c := orphans[0].Spec.Compute; c == nil || c.Backend != "CUDA" || c.Offloaded != 49 {
|
||||
t.Errorf("not kept: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdoptedShardsComputeIsKeptWithoutCrashing(t *testing.T) {
|
||||
s, _ := newSupervisor(t)
|
||||
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20100}
|
||||
if _, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An agent restart: the engine is adopted by a new supervisor, which did
|
||||
// not start it and has no command of its own for it.
|
||||
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
|
||||
&scriptedHealth{}, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
orphans, _ := s2.FindOrphans()
|
||||
if len(orphans) != 1 {
|
||||
t.Fatalf("%d orphans", len(orphans))
|
||||
}
|
||||
if _, err := s2.Adopt(orphans[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s2.Stop(context.Background(), "a") })
|
||||
s2.SetCompute("a", engines.Compute{Backend: "CPU"})
|
||||
|
||||
// And a third agent still finds it, with its command line and compute.
|
||||
s3, _ := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
|
||||
&scriptedHealth{}, slog.New(slog.DiscardHandler))
|
||||
again, _ := s3.FindOrphans()
|
||||
if len(again) != 1 || len(again[0].Cmdline) == 0 || again[0].Spec.Compute == nil {
|
||||
t.Fatalf("lost: %+v", again)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdoptedEngineGoesOnWritingAndItsEarlierOutputIsKept(t *testing.T) {
|
||||
// An engine's output went to a pipe of the agent's. The agent that
|
||||
// restarted and adopted the engine left that pipe with no reader, and
|
||||
// the engine's next write past the pipe's buffer failed with EPIPE —
|
||||
// which, in a Python engine, raised inside whatever was being reported
|
||||
// on and failed the request. Output goes to a file now: a reader is not
|
||||
// something a file can lose, and the lines from before the restart are
|
||||
// there for the agent after it.
|
||||
s, _ := newSupervisor(t)
|
||||
// 256 KiB at once outgrows a pipe's buffer; a reader gone means EPIPE.
|
||||
sh, err := s.Start(t.Context(), "a", engines.ShardSpec{ReplicaID: "r"},
|
||||
shellCommand(`echo before-restart; sleep 1; head -c 262144 /dev/zero | tr '\0' x; echo; echo after-restart-done; sleep 30`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, 3*time.Second, "the first line was never captured", func() bool {
|
||||
return strings.Contains(strings.Join(sh.Logs(10), "\n"), "before-restart")
|
||||
})
|
||||
// The agent goes away without stopping the engine: nothing reads the
|
||||
// output between here and the adoption.
|
||||
pid := sh.Status().PID
|
||||
sh.outOnce.Do(func() { close(sh.outStop) })
|
||||
sh.outDone.Wait()
|
||||
|
||||
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 3 * time.Second, LogLines: 64},
|
||||
&scriptedHealth{}, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
orphans, _ := s2.FindOrphans()
|
||||
adopted, err := s2.Adopt(orphans[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, 5*time.Second, "the engine's output after the restart never arrived: "+strings.Join(adopted.Logs(5), " | "), func() bool {
|
||||
return strings.Contains(strings.Join(adopted.Logs(64), "\n"), "after-restart-done")
|
||||
})
|
||||
if !strings.Contains(strings.Join(adopted.Logs(64), "\n"), "before-restart") {
|
||||
t.Error("the output from before the restart is not in the adopted log")
|
||||
}
|
||||
if !processAlive(pid) {
|
||||
t.Fatal("the engine died writing output nobody was reading")
|
||||
}
|
||||
if err := s2.Stop(t.Context(), "a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(s2.outputPath("a", "")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Errorf("the output file was not removed with the shard: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOutputFileIsEmptiedPastItsBound(t *testing.T) {
|
||||
s, _ := newSupervisor(t)
|
||||
path := s.outputPath("a", "")
|
||||
// A file already past the bound, as a long-running engine leaves one.
|
||||
big, _ := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0o644)
|
||||
_, _ = big.Write(bytes.Repeat([]byte("x"), maxOutputBytes+1))
|
||||
big.Close()
|
||||
sh := &Shard{ID: "a", logs: newRingBuffer(8), done: make(chan struct{}), outStop: make(chan struct{})}
|
||||
sh.outDone.Add(1)
|
||||
go s.follow(sh, path, "")
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
f, _ := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o644)
|
||||
_, _ = f.WriteString("still here\n")
|
||||
f.Close()
|
||||
s.stopOutput(sh)
|
||||
st, _ := os.Stat(path)
|
||||
if st.Size() > 1024 {
|
||||
t.Errorf("the file was not emptied: %d bytes", st.Size())
|
||||
}
|
||||
if !strings.Contains(strings.Join(sh.Logs(8), "\n"), "still here") {
|
||||
t.Errorf("a line written after the file was emptied was lost: %v", sh.Logs(8))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,574 @@
|
||||
// Package train runs LoRA training jobs on a node (BACKLOG F4): with an
|
||||
// installed engine's Python — its torch and transformers, already matched to
|
||||
// the node's CUDA — and a small overlay of what training needs besides, so a
|
||||
// node that serves can train without a second multi-gigabyte environment.
|
||||
package train
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
_ "embed"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
)
|
||||
|
||||
//go:embed trainer.py
|
||||
var trainerPy []byte
|
||||
|
||||
// overlayPackages are what training needs besides an engine's Python, pinned
|
||||
// and installed without their dependencies: those are the engine's own, and
|
||||
// letting an installer resolve them pulled a second torch.
|
||||
var overlayPackages = []string{"peft==0.21.2", "accelerate==1.15.0"}
|
||||
|
||||
// Job is one training job, as the control plane sent it.
|
||||
type Job struct {
|
||||
ID string
|
||||
BaseRepo string
|
||||
BaseRevision string
|
||||
// Data is NDJSON of {"q", "a"}.
|
||||
Data []byte
|
||||
// Params is the trainer's settings as JSON.
|
||||
Params string
|
||||
GPUMemoryMiB int
|
||||
GGUF bool
|
||||
}
|
||||
|
||||
// Status is how a job is going.
|
||||
type Status struct {
|
||||
State string
|
||||
Message string
|
||||
Step int
|
||||
Steps int
|
||||
Loss float64
|
||||
PeakMiB int
|
||||
}
|
||||
|
||||
// Env is where a node keeps what training needs.
|
||||
type Env struct {
|
||||
// EnginesDir is where engines are installed (the agent's engines_dir).
|
||||
EnginesDir string
|
||||
// WorkDir is the trainer's own directory: its overlay, the llama.cpp
|
||||
// conversion code, and each job's files while it runs.
|
||||
WorkDir string
|
||||
// RunAs is the user jobs run as: the engines' user.
|
||||
RunAs string
|
||||
Log *slog.Logger
|
||||
// Python, UV, ConvDir and GPUs replace what would be found, for tests.
|
||||
Python, UV, ConvDir string
|
||||
GPUs func(ctx context.Context) ([]GPU, error)
|
||||
}
|
||||
|
||||
// GPU is one card and how much of it is free.
|
||||
type GPU struct {
|
||||
Index int
|
||||
FreeMiB int
|
||||
}
|
||||
|
||||
// Runner runs one job at a time.
|
||||
type Runner struct {
|
||||
env Env
|
||||
mu sync.Mutex
|
||||
cancel context.CancelFunc
|
||||
job string
|
||||
}
|
||||
|
||||
// New is a runner.
|
||||
func New(env Env) *Runner {
|
||||
if env.Log == nil {
|
||||
env.Log = slog.New(slog.DiscardHandler)
|
||||
}
|
||||
return &Runner{env: env}
|
||||
}
|
||||
|
||||
// Cancel stops a job if it is the one running.
|
||||
func (r *Runner) Cancel(id string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.job == id && r.cancel != nil {
|
||||
r.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
// ErrBusy is a node already training.
|
||||
var ErrBusy = errors.New("this node is already training")
|
||||
|
||||
// Run runs a job to its end, telling status as it goes and artifact each
|
||||
// piece of what it made. It answers the error it failed with.
|
||||
func (r *Runner) Run(ctx context.Context, job Job, status func(Status), artifact func(kind string, seq int, data []byte, last bool) error) error {
|
||||
r.mu.Lock()
|
||||
if r.job != "" {
|
||||
r.mu.Unlock()
|
||||
return ErrBusy
|
||||
}
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
r.job, r.cancel = job.ID, cancel
|
||||
r.mu.Unlock()
|
||||
defer func() {
|
||||
cancel()
|
||||
r.mu.Lock()
|
||||
r.job, r.cancel = "", nil
|
||||
r.mu.Unlock()
|
||||
}()
|
||||
|
||||
status(Status{State: "preparing", Message: "preparing the trainer"})
|
||||
py, err := r.python()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
overlay, err := r.overlay(ctx, py)
|
||||
if err != nil {
|
||||
return fmt.Errorf("installing %s: %w", strings.Join(overlayPackages, ", "), err)
|
||||
}
|
||||
conv := ""
|
||||
if job.GGUF {
|
||||
if conv, err = r.conversion(ctx); err != nil {
|
||||
return fmt.Errorf("fetching llama.cpp's conversion code: %w", err)
|
||||
}
|
||||
}
|
||||
gpu, err := r.pickGPU(ctx, job.GPUMemoryMiB)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dir := filepath.Join(r.env.WorkDir, "jobs", job.ID)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
out := filepath.Join(dir, "adapter")
|
||||
var params map[string]any
|
||||
if job.Params != "" {
|
||||
if err := json.Unmarshal([]byte(job.Params), ¶ms); err != nil {
|
||||
return fmt.Errorf("the job's params: %w", err)
|
||||
}
|
||||
}
|
||||
spec, _ := json.Marshal(map[string]any{"base_repo": job.BaseRepo, "base_revision": job.BaseRevision,
|
||||
"data": filepath.Join(dir, "data.jsonl"), "out": out, "params": params})
|
||||
for name, b := range map[string][]byte{"data.jsonl": job.Data, "job.json": spec, "trainer.py": trainerPy} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), b, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
var baseDir string
|
||||
err = r.python3(ctx, py, dir, []string{"PYTHONPATH=" + overlay, "CUDA_VISIBLE_DEVICES=" + strconv.Itoa(gpu)},
|
||||
[]string{filepath.Join(dir, "trainer.py"), filepath.Join(dir, "job.json")},
|
||||
func(line map[string]any) {
|
||||
s := Status{State: str(line["state"]), Message: str(line["message"]), Step: num(line["step"]),
|
||||
Steps: num(line["steps"]), PeakMiB: num(line["peak_mib"])}
|
||||
if v, ok := line["loss"].(float64); ok {
|
||||
s.Loss = v
|
||||
}
|
||||
if b := str(line["base_dir"]); b != "" {
|
||||
baseDir = b
|
||||
}
|
||||
if s.State == "trained" {
|
||||
s.State = "running"
|
||||
s.Message = "trained; packing the adapter"
|
||||
}
|
||||
status(s)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if job.GGUF {
|
||||
status(Status{State: "running", Message: "converting the adapter to GGUF for llama.cpp"})
|
||||
gguf := filepath.Join(dir, "adapter.gguf")
|
||||
err := r.python3(ctx, py, dir, []string{"PYTHONPATH=" + strings.Join([]string{filepath.Join(conv, "gguf-py"), conv, overlay}, ":")},
|
||||
[]string{filepath.Join(conv, "convert_lora_to_gguf.py"), "--base", baseDir, "--outtype", "f16", "--outfile", gguf, out}, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("converting to GGUF: %w", err)
|
||||
}
|
||||
if err := sendFile(gguf, "gguf", artifact); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var tgz bytes.Buffer
|
||||
if err := tarGz(&tgz, out); err != nil {
|
||||
return err
|
||||
}
|
||||
return sendBytes(tgz.Bytes(), "peft", artifact)
|
||||
}
|
||||
|
||||
func str(v any) string {
|
||||
s, _ := v.(string)
|
||||
return s
|
||||
}
|
||||
|
||||
func num(v any) int {
|
||||
f, _ := v.(float64)
|
||||
return int(f)
|
||||
}
|
||||
|
||||
// python3 runs a Python script as the engines' user, passing each JSON line
|
||||
// it prints to line, and answers the last of what it wrote to stderr when it
|
||||
// fails.
|
||||
func (r *Runner) python3(ctx context.Context, py, dir string, env, args []string, line func(map[string]any)) error {
|
||||
cmd := exec.CommandContext(ctx, py, args...)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = append(os.Environ(), "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True", "TOKENIZERS_PARALLELISM=false",
|
||||
"HF_HUB_DISABLE_PROGRESS_BARS=1", "PYTHONUNBUFFERED=1")
|
||||
cmd.Env = append(cmd.Env, env...)
|
||||
home, give, why := supervisor.AsEngineUser(cmd, r.env.RunAs, r.env.WorkDir)
|
||||
if why != "" {
|
||||
r.env.Log.Warn(why)
|
||||
}
|
||||
if home != "" {
|
||||
cmd.Env = append(cmd.Env, "HF_HOME="+filepath.Join(home, ".cache", "huggingface"))
|
||||
} else {
|
||||
cmd.Env = append(cmd.Env, "HF_HOME="+filepath.Join(r.env.WorkDir, "hf"))
|
||||
}
|
||||
if err := give(dir); err != nil {
|
||||
return fmt.Errorf("handing %s to %s: %w", dir, r.env.RunAs, err)
|
||||
}
|
||||
// The whole process group goes when the job is cancelled: torch starts
|
||||
// workers of its own.
|
||||
if cmd.SysProcAttr == nil {
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
}
|
||||
cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var stderr tailBuffer
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
sc := bufio.NewScanner(stdout)
|
||||
sc.Buffer(make([]byte, 64<<10), 1<<20)
|
||||
for sc.Scan() {
|
||||
var m map[string]any
|
||||
if line != nil && json.Unmarshal(sc.Bytes(), &m) == nil {
|
||||
line(m)
|
||||
}
|
||||
}
|
||||
err = cmd.Wait()
|
||||
if ctx.Err() != nil {
|
||||
return errors.New("cancelled")
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("%v: %s", err, stderr.last())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// tailBuffer keeps the end of what a process wrote: the reason it failed is
|
||||
// at the end, and the rest is progress bars.
|
||||
type tailBuffer struct{ b []byte }
|
||||
|
||||
func (t *tailBuffer) Write(p []byte) (int, error) {
|
||||
t.b = append(t.b, p...)
|
||||
if len(t.b) > 16<<10 {
|
||||
t.b = t.b[len(t.b)-16<<10:]
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (t *tailBuffer) last() string {
|
||||
lines := strings.Split(strings.TrimSpace(string(t.b)), "\n")
|
||||
if len(lines) > 3 {
|
||||
lines = lines[len(lines)-3:]
|
||||
}
|
||||
return strings.Join(lines, " | ")
|
||||
}
|
||||
|
||||
// python is the newest installed vLLM's interpreter: its torch is built for
|
||||
// this node's CUDA.
|
||||
func (r *Runner) python() (string, error) {
|
||||
if r.env.Python != "" {
|
||||
return r.env.Python, nil
|
||||
}
|
||||
found, _ := filepath.Glob(filepath.Join(r.env.EnginesDir, "vllm", "*", "bin", "python3"))
|
||||
if len(found) == 0 {
|
||||
return "", errors.New("training needs vLLM installed on this node (its torch is what trains); " +
|
||||
"llmgw ctl engine-build, or serving any vLLM model here, installs it")
|
||||
}
|
||||
sort.Strings(found)
|
||||
return found[len(found)-1], nil
|
||||
}
|
||||
|
||||
func (r *Runner) uv() (string, error) {
|
||||
if r.env.UV != "" {
|
||||
return r.env.UV, nil
|
||||
}
|
||||
var found []string
|
||||
_ = filepath.WalkDir(filepath.Join(r.env.EnginesDir, "uv"), func(p string, d os.DirEntry, err error) error {
|
||||
if err == nil && !d.IsDir() && d.Name() == "uv" {
|
||||
found = append(found, p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if len(found) == 0 {
|
||||
return "", errors.New("no uv under " + filepath.Join(r.env.EnginesDir, "uv"))
|
||||
}
|
||||
sort.Strings(found)
|
||||
return found[len(found)-1], nil
|
||||
}
|
||||
|
||||
// overlay installs the pinned packages once per Python and set of pins.
|
||||
func (r *Runner) overlay(ctx context.Context, py string) (string, error) {
|
||||
sum := sha256.Sum256([]byte(py + "\x00" + strings.Join(overlayPackages, " ")))
|
||||
dir := filepath.Join(r.env.WorkDir, "overlay-"+hex.EncodeToString(sum[:6]))
|
||||
if _, err := os.Stat(filepath.Join(dir, ".done")); err == nil {
|
||||
return dir, nil
|
||||
}
|
||||
uv, err := r.uv()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
tmp := dir + ".new"
|
||||
_ = os.RemoveAll(tmp)
|
||||
args := append([]string{"pip", "install", "--quiet", "--python", py, "--no-deps", "--target", tmp}, overlayPackages...)
|
||||
cmd := exec.CommandContext(ctx, uv, args...)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf("%v: %s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
_ = os.WriteFile(filepath.Join(tmp, ".done"), nil, 0o644)
|
||||
_ = os.RemoveAll(dir)
|
||||
if err := os.Rename(tmp, dir); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Readable by the engines' user, which runs the trainer.
|
||||
_ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
|
||||
if err == nil {
|
||||
mode := os.FileMode(0o644)
|
||||
if d.IsDir() {
|
||||
mode = 0o755
|
||||
}
|
||||
_ = os.Chmod(p, mode)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return dir, nil
|
||||
}
|
||||
|
||||
// llamaCppTag is the release of the newest llama.cpp installed: its
|
||||
// conversion code is the one whose GGUF that llama.cpp reads.
|
||||
func (r *Runner) llamaCppTag() (string, error) {
|
||||
found, _ := filepath.Glob(filepath.Join(r.env.EnginesDir, "llamacpp", "b*"))
|
||||
if len(found) == 0 {
|
||||
return "", errors.New("a GGUF adapter needs llama.cpp installed on this node, to match its release")
|
||||
}
|
||||
sort.Strings(found)
|
||||
tag, _, _ := strings.Cut(filepath.Base(found[len(found)-1]), "-")
|
||||
return tag, nil
|
||||
}
|
||||
|
||||
// conversionSource is where llama.cpp's source is fetched from, by tag.
|
||||
var conversionSource = "https://codeload.github.com/ggml-org/llama.cpp/tar.gz/refs/tags/"
|
||||
|
||||
// conversion fetches llama.cpp's LoRA conversion code for the installed
|
||||
// release, once.
|
||||
func (r *Runner) conversion(ctx context.Context) (string, error) {
|
||||
if r.env.ConvDir != "" {
|
||||
return r.env.ConvDir, nil
|
||||
}
|
||||
tag, err := r.llamaCppTag()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
dir := filepath.Join(r.env.WorkDir, "llama.cpp-"+tag)
|
||||
if _, err := os.Stat(filepath.Join(dir, "convert_lora_to_gguf.py")); err == nil {
|
||||
return dir, nil
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, conversionSource+tag, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("%s answered %s", conversionSource+tag, res.Status)
|
||||
}
|
||||
tmp := dir + ".new"
|
||||
_ = os.RemoveAll(tmp)
|
||||
if err := untarConversion(res.Body, tmp); err != nil {
|
||||
return "", err
|
||||
}
|
||||
_ = os.RemoveAll(dir)
|
||||
return dir, os.Rename(tmp, dir)
|
||||
}
|
||||
|
||||
// untarConversion keeps, of llama.cpp's source, the conversion scripts and
|
||||
// the Python packages they import.
|
||||
func untarConversion(r io.Reader, dir string) error {
|
||||
gz, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tr := tar.NewReader(gz)
|
||||
kept := 0
|
||||
for {
|
||||
h, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, rel, ok := strings.Cut(h.Name, "/")
|
||||
if !ok || h.Typeflag != tar.TypeReg || strings.Contains(rel, "..") {
|
||||
continue
|
||||
}
|
||||
if rel != "convert_lora_to_gguf.py" && rel != "convert_hf_to_gguf.py" &&
|
||||
!strings.HasPrefix(rel, "conversion/") && !strings.HasPrefix(rel, "gguf-py/") {
|
||||
continue
|
||||
}
|
||||
p := filepath.Join(dir, filepath.FromSlash(rel))
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.OpenFile(p, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(f, io.LimitReader(tr, 64<<20))
|
||||
f.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kept++
|
||||
}
|
||||
if kept == 0 {
|
||||
return errors.New("no conversion code in llama.cpp's source")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pickGPU is the card with the most memory free, if that is enough.
|
||||
func (r *Runner) pickGPU(ctx context.Context, need int) (int, error) {
|
||||
list := r.env.GPUs
|
||||
if list == nil {
|
||||
list = nvidiaSMI
|
||||
}
|
||||
gpus, err := list(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("reading the cards' free memory: %w", err)
|
||||
}
|
||||
if len(gpus) == 0 {
|
||||
return 0, errors.New("this node has no GPU to train on")
|
||||
}
|
||||
best := gpus[0]
|
||||
for _, g := range gpus[1:] {
|
||||
if g.FreeMiB > best.FreeMiB {
|
||||
best = g
|
||||
}
|
||||
}
|
||||
if best.FreeMiB < need {
|
||||
return 0, fmt.Errorf("no card here has %d MiB free (the most free is GPU %d with %d MiB); "+
|
||||
"a smaller base, or a card freed for it, would do", need, best.Index, best.FreeMiB)
|
||||
}
|
||||
return best.Index, nil
|
||||
}
|
||||
|
||||
func nvidiaSMI(ctx context.Context) ([]GPU, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, "nvidia-smi", "--query-gpu=index,memory.free", "--format=csv,noheader,nounits").Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseGPUs(string(out)), nil
|
||||
}
|
||||
|
||||
func parseGPUs(s string) []GPU {
|
||||
var out []GPU
|
||||
for _, line := range strings.Split(strings.TrimSpace(s), "\n") {
|
||||
a, b, ok := strings.Cut(line, ",")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
i, err1 := strconv.Atoi(strings.TrimSpace(a))
|
||||
f, err2 := strconv.Atoi(strings.TrimSpace(b))
|
||||
if err1 == nil && err2 == nil {
|
||||
out = append(out, GPU{Index: i, FreeMiB: f})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tarGz packs a directory.
|
||||
func tarGz(w io.Writer, dir string) error {
|
||||
gz := gzip.NewWriter(w)
|
||||
tw := tar.NewWriter(gz)
|
||||
err := filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(dir, p)
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tw.WriteHeader(&tar.Header{Name: filepath.ToSlash(rel), Mode: 0o644, Size: info.Size(), ModTime: info.ModTime()}); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.Open(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
_, err = io.Copy(tw, f)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return gz.Close()
|
||||
}
|
||||
|
||||
// artifactChunk is how much of an artifact goes in one message.
|
||||
const artifactChunk = 1 << 20
|
||||
|
||||
func sendFile(path, kind string, send func(string, int, []byte, bool) error) error {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return sendBytes(b, kind, send)
|
||||
}
|
||||
|
||||
func sendBytes(b []byte, kind string, send func(string, int, []byte, bool) error) error {
|
||||
seq := 0
|
||||
for {
|
||||
n := min(artifactChunk, len(b))
|
||||
last := n == len(b)
|
||||
if err := send(kind, seq, b[:n], last); err != nil {
|
||||
return err
|
||||
}
|
||||
if last {
|
||||
return nil
|
||||
}
|
||||
b = b[n:]
|
||||
seq++
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package train
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakePython stands in for an engine's interpreter: as the trainer it
|
||||
// prints progress and writes an adapter; as the converter it writes a GGUF;
|
||||
// told to fail, it fails.
|
||||
const fakePython = `#!/bin/sh
|
||||
case "$1" in
|
||||
*trainer.py)
|
||||
out=$(sed -n 's/.*"out": *"\([^"]*\)".*/\1/p' "$2")
|
||||
grep -q FAIL "$(dirname "$2")/data.jsonl" && { echo "CUDA out of memory" >&2; exit 1; }
|
||||
echo '{"state":"preparing","message":"fetching base"}'
|
||||
echo '{"state":"running","step":5,"steps":10,"loss":0.9,"peak_mib":1900}'
|
||||
mkdir -p "$out"; echo '{"r":16}' > "$out/adapter_config.json"; echo weights > "$out/adapter_model.safetensors"
|
||||
echo '{"state":"trained","step":10,"steps":10,"peak_mib":2100,"base_dir":"/base"}'
|
||||
;;
|
||||
*convert_lora_to_gguf.py)
|
||||
while [ $# -gt 0 ]; do [ "$1" = "--outfile" ] && echo GGUF > "$2"; shift; done
|
||||
;;
|
||||
esac
|
||||
`
|
||||
|
||||
func runner(t *testing.T, free int) *Runner {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
py := filepath.Join(dir, "python3")
|
||||
_ = os.WriteFile(py, []byte(fakePython), 0o755)
|
||||
uv := filepath.Join(dir, "uv")
|
||||
_ = os.WriteFile(uv, []byte("#!/bin/sh\nwhile [ $# -gt 0 ]; do [ \"$1\" = \"--target\" ] && mkdir -p \"$2\"; shift; done\n"), 0o755)
|
||||
return New(Env{WorkDir: filepath.Join(dir, "work"), Python: py, UV: uv, ConvDir: dir,
|
||||
GPUs: func(context.Context) ([]GPU, error) { return []GPU{{0, 500}, {1, free}}, nil }})
|
||||
}
|
||||
|
||||
type sink struct {
|
||||
mu sync.Mutex
|
||||
statuses []Status
|
||||
parts map[string][]byte
|
||||
lasts int
|
||||
}
|
||||
|
||||
func (s *sink) status(st Status) { s.mu.Lock(); s.statuses = append(s.statuses, st); s.mu.Unlock() }
|
||||
func (s *sink) artifact(kind string, _ int, b []byte, last bool) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.parts == nil {
|
||||
s.parts = map[string][]byte{}
|
||||
}
|
||||
s.parts[kind] = append(s.parts[kind], b...)
|
||||
if last {
|
||||
s.lasts++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAJobTrainsOnTheFreestCardAndSendsWhatItMade(t *testing.T) {
|
||||
r := runner(t, 3000)
|
||||
s := &sink{}
|
||||
err := r.Run(context.Background(), Job{ID: "j1", BaseRepo: "Qwen/x", Data: []byte(`{"q":"a","a":"b"}`), GPUMemoryMiB: 2500, GGUF: true},
|
||||
s.status, s.artifact)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var running bool
|
||||
for _, st := range s.statuses {
|
||||
if st.State == "running" && st.Step == 5 && st.Steps == 10 && st.PeakMiB == 1900 {
|
||||
running = true
|
||||
}
|
||||
}
|
||||
if !running {
|
||||
t.Errorf("statuses %+v", s.statuses)
|
||||
}
|
||||
if string(s.parts["gguf"]) != "GGUF\n" || s.lasts != 2 {
|
||||
t.Errorf("gguf %q, lasts %d", s.parts["gguf"], s.lasts)
|
||||
}
|
||||
files := map[string]string{}
|
||||
gz, err := gzip.NewReader(bytes.NewReader(s.parts["peft"]))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tr := tar.NewReader(gz)
|
||||
for {
|
||||
h, err := tr.Next()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
b, _ := io.ReadAll(tr)
|
||||
files[h.Name] = string(b)
|
||||
}
|
||||
if files["adapter_model.safetensors"] != "weights\n" || files["adapter_config.json"] == "" {
|
||||
t.Errorf("peft archive %v", files)
|
||||
}
|
||||
// Nothing of the job is left on the node.
|
||||
if entries, _ := os.ReadDir(filepath.Join(r.env.WorkDir, "jobs")); len(entries) != 0 {
|
||||
t.Errorf("left behind: %v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAJobWaitsForNoCardItDoesNotFit(t *testing.T) {
|
||||
r := runner(t, 1000)
|
||||
err := r.Run(context.Background(), Job{ID: "j1", Data: []byte(`{"q":"a","a":"b"}`), GPUMemoryMiB: 2500}, (&sink{}).status, (&sink{}).artifact)
|
||||
if err == nil || !strings.Contains(err.Error(), "no card here has 2500 MiB free") {
|
||||
t.Errorf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATrainerThatFailsSaysWhy(t *testing.T) {
|
||||
r := runner(t, 3000)
|
||||
err := r.Run(context.Background(), Job{ID: "j1", Data: []byte(`{"q":"FAIL","a":"b"}`)}, (&sink{}).status, (&sink{}).artifact)
|
||||
if err == nil || !strings.Contains(err.Error(), "CUDA out of memory") {
|
||||
t.Errorf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneJobAtATime(t *testing.T) {
|
||||
r := runner(t, 3000)
|
||||
r.job = "other"
|
||||
if err := r.Run(context.Background(), Job{ID: "j2"}, (&sink{}).status, (&sink{}).artifact); err != ErrBusy {
|
||||
t.Errorf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCardsAreReadFromNvidiaSMI(t *testing.T) {
|
||||
got := parseGPUs("0, 2310\n1, 80000\nbad line\n")
|
||||
if len(got) != 2 || got[1].FreeMiB != 80000 || got[0].Index != 0 {
|
||||
t.Errorf("parsed %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheConversionCodeIsKeptOfLlamaCpp(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(gz)
|
||||
for _, name := range []string{"llama.cpp-b1/convert_lora_to_gguf.py", "llama.cpp-b1/conversion/qwen.py",
|
||||
"llama.cpp-b1/gguf-py/gguf/__init__.py", "llama.cpp-b1/src/llama.cpp", "llama.cpp-b1/../evil"} {
|
||||
_ = tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: 1, Typeflag: tar.TypeReg})
|
||||
_, _ = tw.Write([]byte("x"))
|
||||
}
|
||||
_ = tw.Close()
|
||||
_ = gz.Close()
|
||||
dir := filepath.Join(t.TempDir(), "conv")
|
||||
if err := untarConversion(&buf, dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"convert_lora_to_gguf.py", "conversion/qwen.py", "gguf-py/gguf/__init__.py"} {
|
||||
if _, err := os.Stat(filepath.Join(dir, want)); err != nil {
|
||||
t.Errorf("%s not kept", want)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "src")); err == nil {
|
||||
t.Error("the rest of llama.cpp was kept")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
# llmgw's LoRA trainer (BACKLOG F4), run by the node agent with an installed
|
||||
# engine's Python (its torch and transformers) and a small overlay (peft,
|
||||
# accelerate). Reads a job, trains, writes the PEFT adapter, and says how it
|
||||
# goes as one JSON object a line on standard output.
|
||||
import json, math, os, random, sys, threading, subprocess, time
|
||||
|
||||
def say(**kw):
|
||||
print(json.dumps(kw), flush=True)
|
||||
|
||||
job = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
p = {"rank": 16, "alpha": 32, "dropout": 0.05, "epochs": 6, "lr": 2e-4, "batch": 4, "max_len": 512, "seed": 0}
|
||||
p.update(job.get("params") or {})
|
||||
|
||||
say(state="preparing", message="loading libraries")
|
||||
import torch
|
||||
from huggingface_hub import snapshot_download
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
from peft import LoraConfig, get_peft_model
|
||||
|
||||
random.seed(p["seed"]); torch.manual_seed(p["seed"])
|
||||
if not torch.cuda.is_available():
|
||||
raise SystemExit("no CUDA device is visible to the trainer")
|
||||
|
||||
say(state="preparing", message="fetching " + job["base_repo"])
|
||||
base_dir = snapshot_download(job["base_repo"], revision=job.get("base_revision") or None,
|
||||
allow_patterns=["*.json", "*.safetensors", "*.txt", "*.model", "*.tiktoken"])
|
||||
say(state="preparing", message="loading the base model", base_dir=base_dir)
|
||||
|
||||
# bf16 where the card has it; fp16 with a loss scaler where it does not
|
||||
# (Turing and older).
|
||||
bf16 = torch.cuda.is_bf16_supported()
|
||||
dtype = torch.bfloat16 if bf16 else torch.float16
|
||||
tok = AutoTokenizer.from_pretrained(base_dir)
|
||||
if tok.pad_token_id is None:
|
||||
tok.pad_token = tok.eos_token
|
||||
model = AutoModelForCausalLM.from_pretrained(base_dir, dtype=dtype).to("cuda")
|
||||
|
||||
rows = [json.loads(l) for l in open(job["data"], encoding="utf-8") if l.strip()]
|
||||
rows = [r for r in rows if r.get("q") and r.get("a")]
|
||||
if not rows:
|
||||
raise SystemExit("no {q, a} pairs in the data")
|
||||
|
||||
def encode(r):
|
||||
prompt = tok.apply_chat_template([{"role": "user", "content": r["q"]}], add_generation_prompt=True, tokenize=False)
|
||||
full = tok.apply_chat_template([{"role": "user", "content": r["q"]}, {"role": "assistant", "content": r["a"]}], tokenize=False)
|
||||
pi = tok(prompt, add_special_tokens=False)["input_ids"]
|
||||
fi = tok(full, add_special_tokens=False)["input_ids"][: p["max_len"]]
|
||||
return fi, ([-100] * len(pi) + fi[len(pi):])[: len(fi)]
|
||||
|
||||
data = [encode(r) for r in rows]
|
||||
|
||||
model.gradient_checkpointing_enable()
|
||||
model.enable_input_require_grads()
|
||||
model = get_peft_model(model, LoraConfig(r=int(p["rank"]), lora_alpha=int(p["alpha"]), lora_dropout=float(p["dropout"]),
|
||||
task_type="CAUSAL_LM", target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]))
|
||||
trainable = [x for x in model.parameters() if x.requires_grad]
|
||||
for x in trainable: # fp32 trainable weights, so a loss scaler can unscale them
|
||||
x.data = x.data.float()
|
||||
|
||||
peak = {"mib": 0}
|
||||
stop = threading.Event()
|
||||
def watch():
|
||||
while not stop.is_set():
|
||||
try:
|
||||
out = subprocess.run(["nvidia-smi", "--query-compute-apps=pid,used_memory", "--format=csv,noheader,nounits"],
|
||||
capture_output=True, text=True, timeout=5).stdout
|
||||
for line in out.splitlines():
|
||||
pid, mib = [v.strip() for v in line.split(",")]
|
||||
if int(pid) == os.getpid():
|
||||
peak["mib"] = max(peak["mib"], int(mib))
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
threading.Thread(target=watch, daemon=True).start()
|
||||
|
||||
opt = torch.optim.AdamW(trainable, lr=float(p["lr"]), weight_decay=0.0)
|
||||
scaler = torch.amp.GradScaler("cuda", enabled=not bf16)
|
||||
bs, epochs = int(p["batch"]), int(p["epochs"])
|
||||
steps = epochs * math.ceil(len(data) / bs)
|
||||
warm = max(1, steps // 10)
|
||||
sched = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: min(1.0, (s + 1) / warm) * max(0.0, 1 - s / steps))
|
||||
model.train(); step = 0; last = time.time()
|
||||
for ep in range(epochs):
|
||||
random.shuffle(data)
|
||||
for i in range(0, len(data), bs):
|
||||
batch = data[i:i + bs]; n = max(len(x[0]) for x in batch)
|
||||
ids = torch.tensor([x[0] + [tok.pad_token_id] * (n - len(x[0])) for x in batch], device="cuda")
|
||||
lab = torch.tensor([x[1] + [-100] * (n - len(x[1])) for x in batch], device="cuda")
|
||||
att = torch.tensor([[1] * len(x[0]) + [0] * (n - len(x[0])) for x in batch], device="cuda")
|
||||
with torch.autocast("cuda", dtype=dtype):
|
||||
loss = model(input_ids=ids, attention_mask=att, labels=lab).loss
|
||||
scaler.scale(loss).backward(); scaler.unscale_(opt)
|
||||
torch.nn.utils.clip_grad_norm_(trainable, 1.0)
|
||||
scaler.step(opt); scaler.update(); opt.zero_grad(set_to_none=True); sched.step(); step += 1
|
||||
if time.time() - last > 2 or step == steps:
|
||||
last = time.time()
|
||||
say(state="running", message="training", step=step, steps=steps, loss=round(loss.item(), 4), peak_mib=peak["mib"])
|
||||
|
||||
model.save_pretrained(job["out"])
|
||||
stop.set()
|
||||
say(state="trained", step=step, steps=steps, peak_mib=peak["mib"], base_dir=base_dir)
|
||||
@@ -0,0 +1,46 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/train"
|
||||
)
|
||||
|
||||
// trainJob runs a training job the control plane sent, telling it how the
|
||||
// job goes and sending what it made. A stream that ends ends the job: the
|
||||
// control plane, which sees the node go, marks it failed, and it is started
|
||||
// again from the beginning rather than resumed.
|
||||
func (a *Agent) trainJob(ctx context.Context, m *llmgwv1.TrainStart, out chan<- *llmgwv1.AgentMessage) {
|
||||
id := m.GetJobId()
|
||||
send := func(msg *llmgwv1.AgentMessage) bool {
|
||||
select {
|
||||
case out <- msg:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
status := func(s train.Status) {
|
||||
send(&llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_TrainStatus{TrainStatus: &llmgwv1.TrainStatus{
|
||||
JobId: id, State: s.State, Message: s.Message, Step: int32(s.Step), Steps: int32(s.Steps), //nolint:gosec // counts
|
||||
Loss: s.Loss, PeakMib: int32(s.PeakMiB)}}}) //nolint:gosec // MiB
|
||||
}
|
||||
artifact := func(kind string, seq int, data []byte, last bool) error {
|
||||
if !send(&llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_TrainArtifact{TrainArtifact: &llmgwv1.TrainArtifact{
|
||||
JobId: id, Kind: kind, Seq: int32(seq), Data: data, Last: last}}}) { //nolint:gosec // chunk count
|
||||
return ctx.Err()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
a.log.Info("training an adapter", "job", id, "base", m.GetBaseRepo())
|
||||
err := a.trainer.Run(ctx, train.Job{ID: id, BaseRepo: m.GetBaseRepo(), BaseRevision: m.GetBaseRevision(),
|
||||
Data: m.GetData(), Params: m.GetParams(), GPUMemoryMiB: int(m.GetGpuMemoryMib()), GGUF: m.GetGguf()}, status, artifact)
|
||||
if err != nil {
|
||||
a.log.Warn("training failed", "job", id, "err", err)
|
||||
status(train.Status{State: "failed", Message: err.Error()})
|
||||
return
|
||||
}
|
||||
a.log.Info("trained an adapter", "job", id)
|
||||
status(train.Status{State: "done"})
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
)
|
||||
|
||||
type alwaysReady struct{}
|
||||
@@ -40,7 +41,7 @@ func TestAnAdoptedEngineThatDiesIsReported(t *testing.T) {
|
||||
out := make(chan *llmgwv1.AgentMessage, 4)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
go a.watchExits(ctx, out)
|
||||
go a.watchExits(ctx, out, nil)
|
||||
time.Sleep(1500 * time.Millisecond) // seen ready
|
||||
_ = cmd.Process.Kill()
|
||||
|
||||
@@ -54,3 +55,50 @@ func TestAnAdoptedEngineThatDiesIsReported(t *testing.T) {
|
||||
t.Fatal("the adopted engine died and nothing was reported")
|
||||
}
|
||||
}
|
||||
|
||||
// Registered while its health check still ran: the ready that follows is
|
||||
// reported, or the control plane keeps the replica starting while it serves.
|
||||
func TestAShardRegisteredStartingIsReportedReady(t *testing.T) {
|
||||
sup, err := supervisor.New(supervisor.Config{WorkDir: t.TempDir(), RunAs: "-"}, alwaysReady{}, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := exec.Command("sleep", "60")
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }()
|
||||
sh, err := sup.Adopt(supervisor.Orphan{ShardID: "m-r0", ReplicaID: "m", PID: cmd.Process.Pid,
|
||||
Spec: engines.ShardSpec{ReplicaID: "m", ShardID: "m-r0", HTTPPort: 1}, Alive: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
a := &Agent{sup: sup, log: slog.New(slog.DiscardHandler)}
|
||||
pending := unready(a.runningShards())
|
||||
if !pending["m-r0"] {
|
||||
t.Fatalf("registration reported it ready already: %v", pending)
|
||||
}
|
||||
out := make(chan *llmgwv1.AgentMessage, 4)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
go a.watchExits(ctx, out, pending)
|
||||
if err := sup.WaitReady(t.Context(), sh, "http://127.0.0.1:1", "m", 5*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
select {
|
||||
case m := <-out:
|
||||
s := m.GetShard()
|
||||
if s.GetShardId() != "m-r0" || s.GetState() != agentsvc.StateToWire(supervisor.Ready) {
|
||||
t.Errorf("reported %+v", s)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("it became ready and nothing was reported")
|
||||
}
|
||||
select {
|
||||
case m := <-out:
|
||||
t.Errorf("reported twice: %+v", m.GetShard())
|
||||
case <-time.After(2500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
+22
-1
@@ -40,10 +40,23 @@ func newAgentCmd() *cobra.Command {
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
log := logger()
|
||||
|
||||
cfg, err := config.LoadAgent(configPath)
|
||||
// The cluster's settings for this agent over its file, as the
|
||||
// control plane last sent them. Settings that no longer load
|
||||
// beside the file are set aside rather than keeping the agent
|
||||
// from starting: the control plane sends them again, and says
|
||||
// why they are refused.
|
||||
settings := agent.ReadSettings()
|
||||
cfg, err := config.LoadAgentWith(configPath, settings)
|
||||
if err != nil && settings != nil {
|
||||
log.Warn("the cluster's settings kept for this agent do not load beside its agent.yaml; starting without them",
|
||||
"err", err)
|
||||
settings = nil
|
||||
cfg, err = config.LoadAgent(configPath)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.Settings = string(settings)
|
||||
if server != "" {
|
||||
cfg.Server = server
|
||||
}
|
||||
@@ -89,9 +102,17 @@ func newAgentCmd() *cobra.Command {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.SetSettingsCheck(func(b []byte) error {
|
||||
_, err := config.LoadAgentWith(configPath, b)
|
||||
return err
|
||||
})
|
||||
|
||||
ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
go a.WatchFile(ctx, configPath, func() error {
|
||||
_, err := config.LoadAgentWith(configPath, agent.ReadSettings())
|
||||
return err
|
||||
})
|
||||
|
||||
log.Info("agent starting",
|
||||
"node", cfg.NodeID, "server", cfg.Server,
|
||||
|
||||
@@ -249,7 +249,14 @@ func pgEnv(dsn string) ([]string, error) {
|
||||
set("PGPASSWORD", pw)
|
||||
}
|
||||
set("PGDATABASE", strings.TrimPrefix(u.Path, "/"))
|
||||
set("PGSSLMODE", u.Query().Get("sslmode"))
|
||||
// The TLS settings go with the mode: verify-full without the cluster's
|
||||
// authority sent libpq looking for ~/.postgresql/root.crt, and the dump
|
||||
// failed on every control plane whose store is replicated.
|
||||
q := u.Query()
|
||||
set("PGSSLMODE", q.Get("sslmode"))
|
||||
set("PGSSLROOTCERT", q.Get("sslrootcert"))
|
||||
set("PGSSLCERT", q.Get("sslcert"))
|
||||
set("PGSSLKEY", q.Get("sslkey"))
|
||||
// A standby if there is one, else whichever answers — not the address's
|
||||
// read-write: a dump only reads, it need not load the primary, and one
|
||||
// that insists on the primary fails exactly while the cluster is failing
|
||||
|
||||
@@ -238,3 +238,17 @@ func TestAStoreOfSeveralHostsIsGivenToPgDumpAsLibpqLists(t *testing.T) {
|
||||
t.Error("an empty host in the list was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPgDumpIsGivenTheClusterAuthorityAlongWithVerifyFull(t *testing.T) {
|
||||
// Without it libpq looked for ~/.postgresql/root.crt and the dump failed.
|
||||
env, err := pgEnv("postgres://llmgw:p@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=verify-full&sslrootcert=%2Fetc%2Fllmgw%2Fpki%2Fca.crt&target_session_attrs=read-write")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := strings.Join(env, " ")
|
||||
for _, want := range []string{"PGSSLMODE=verify-full", "PGSSLROOTCERT=/etc/llmgw/pki/ca.crt"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("%s is not in %s", want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -237,8 +237,16 @@ func (w *settingsWatch) check(ctx context.Context) {
|
||||
// again: what it reads at every use, or not at all.
|
||||
var liveSettings = []string{
|
||||
"guard",
|
||||
// Stopping and starting the service (llmgw stop, llmgw start).
|
||||
"serving",
|
||||
// Sent to the nodes and this machine's firewall unit as they change.
|
||||
"firewall", "firewall_networks",
|
||||
// Sent to the nodes' agents as they change.
|
||||
"agents",
|
||||
// Read by the leader's presets pass each time it runs.
|
||||
"presets",
|
||||
// Handed to the reconciler as they change.
|
||||
"on_demand",
|
||||
// Read by server init, node add and backups, not by the server.
|
||||
"backup_recipients", "backup_copy_to",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
|
||||
)
|
||||
|
||||
// comfyKeyEnv carries the replica's key to the gate: the environment, which
|
||||
// only the engine's user and root can read, rather than the command line,
|
||||
// which every user on the node can.
|
||||
const comfyKeyEnv = "LLMGW_ENGINE_KEY"
|
||||
|
||||
func newComfyUIGateCmd() *cobra.Command {
|
||||
var cfg comfy.GateConfig
|
||||
cmd := &cobra.Command{
|
||||
Use: "comfyui-gate [flags] [-- main.py flags]",
|
||||
Short: "Run ComfyUI as an engine: behind a key, on the loopback (started by the agent)",
|
||||
Hidden: true,
|
||||
Long: "Started by the node agent for a replica whose engine is comfyui. ComfyUI asks\n" +
|
||||
"nobody for a key, so it listens on the loopback only and this gate on the\n" +
|
||||
"replica's port, letting through, with the replica's key, only what the router\n" +
|
||||
"asks of it. It also lays the fetched weights out where ComfyUI looks for them.",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
cfg.Key = os.Getenv(comfyKeyEnv)
|
||||
_ = os.Unsetenv(comfyKeyEnv)
|
||||
cfg.Args = args
|
||||
cfg.Log = logger()
|
||||
return comfy.RunGate(ctx, cfg)
|
||||
},
|
||||
}
|
||||
f := cmd.Flags()
|
||||
f.StringVar(&cfg.Listen, "listen", "", "the replica's address")
|
||||
f.StringVar(&cfg.Model, "model", "", "the model's name, for /v1/models")
|
||||
f.StringVar(&cfg.Weights, "weights", "", "the fetched weights: a file or a directory")
|
||||
f.StringVar(&cfg.State, "state", "", "the replica's own directory")
|
||||
f.StringVar(&cfg.Python, "python", "python3", "the interpreter of ComfyUI's environment")
|
||||
f.StringVar(&cfg.Dir, "comfyui", "", "ComfyUI's directory, where main.py is")
|
||||
for _, n := range []string{"listen", "weights", "state", "comfyui"} {
|
||||
_ = cmd.MarkFlagRequired(n)
|
||||
}
|
||||
return cmd
|
||||
}
|
||||
+59
-7
@@ -116,6 +116,11 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any)
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
// A stream, as it comes: NDJSON a long run writes line by line.
|
||||
if w, ok := out.(io.Writer); ok {
|
||||
_, err := io.Copy(w, resp.Body)
|
||||
return err
|
||||
}
|
||||
// An older control plane answers a path it does not know with its
|
||||
// dashboard: a page, which read as JSON said only "invalid character '<'".
|
||||
if strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
|
||||
@@ -173,13 +178,16 @@ func newCtlCmd() *cobra.Command {
|
||||
newRetryCmd(client),
|
||||
newUpgradeCmd(client),
|
||||
newJobsCmd(client, &asJSON),
|
||||
newMeshJobsCmd(client, &asJSON),
|
||||
newEngineCheckCmd(),
|
||||
newEngineBuildCmd(client),
|
||||
newToolCheckCmd(),
|
||||
newToolCheckCmd(client),
|
||||
newWeightsCmd(client, &asJSON),
|
||||
newCapacityCmd(client, &asJSON),
|
||||
newQuotaCmd(client, &asJSON),
|
||||
newGuardCmd(client, &asJSON),
|
||||
newKBCmd(client, &asJSON),
|
||||
newTrainCmd(client, &asJSON),
|
||||
newConfigCmd(client, &asJSON),
|
||||
newAuditCmd(client, &asJSON),
|
||||
newNotifyCmd(client, &asJSON),
|
||||
@@ -232,8 +240,8 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
// What the node can run, which is the first thing to look
|
||||
// at when a model will not place and the hardware looks
|
||||
// fine.
|
||||
join(n.Engines),
|
||||
classesColumn(n), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
|
||||
enginesColumn(n),
|
||||
classesColumn(n), fabricsColumn(n), n.Replicas, n.LastSeenAgo)
|
||||
}
|
||||
_ = w.Flush()
|
||||
|
||||
@@ -324,6 +332,9 @@ func modelStatus(m adminapi.ModelView) string {
|
||||
return "stopped"
|
||||
case m.Ready >= m.Min && m.Min > 0:
|
||||
return "ready"
|
||||
case m.Min == 0 && m.Ready > 0 && m.Ready >= m.Total:
|
||||
// Asked for, and up until it goes unused for long enough.
|
||||
return "ready (on demand)"
|
||||
case m.Total > m.Ready:
|
||||
return "starting"
|
||||
default:
|
||||
@@ -450,7 +461,7 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
}
|
||||
|
||||
w := tab()
|
||||
fmt.Fprintln(w, "REPLICA\tMODEL\tSTATE\tPOOL\tNODES\tAGE\tIDLE\tENDPOINT")
|
||||
fmt.Fprintln(w, "REPLICA\tMODEL\tSTATE\tPOOL\tON\tNODES\tAGE\tIDLE\tENDPOINT")
|
||||
for _, r := range reps {
|
||||
var nodes []string
|
||||
for _, s := range r.Shards {
|
||||
@@ -468,8 +479,14 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
if r.Restart != "" {
|
||||
state += " (restart " + r.Restart + ")"
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||
r.ID, r.Model, state, r.Pool, join(nodes),
|
||||
// What the engine computes on, by its own account: the first
|
||||
// shard's, which serves.
|
||||
on := "-"
|
||||
if len(r.Shards) > 0 {
|
||||
on = r.Shards[0].Compute.Short()
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||
r.ID, r.Model, state, r.Pool, on, join(nodes),
|
||||
(time.Duration(r.AgeSec) * time.Second).String(),
|
||||
idleColumn(r.IdleSec), r.Endpoint)
|
||||
// A line rather than a column, because most replicas are not
|
||||
@@ -479,7 +496,7 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
if s.Fetch == nil {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(w, " %s: fetching %s\t\t\t\t\t\t\t\n",
|
||||
fmt.Fprintf(w, " %s: fetching %s\t\t\t\t\t\t\t\t\n",
|
||||
s.Node, fetchColumn(s.Fetch))
|
||||
}
|
||||
}
|
||||
@@ -590,7 +607,42 @@ func emitJSON(v any) error {
|
||||
// more than one or MIG instances among them: a card cut into two 3g.40gb
|
||||
// instances beside a whole card is "a100-80g-mig-3g.40gb×2,a100-80g×1", where
|
||||
// the GPUS column's 3 alone would read as three of something.
|
||||
// fabricsColumn is the node's fabrics, and when they are not all up, which
|
||||
// adapters are down: what keeps it out of replicas spanning nodes.
|
||||
func fabricsColumn(n adminapi.NodeView) string {
|
||||
s := join(n.Fabrics)
|
||||
if n.MultiNode != "" && n.MultiNode != "ready" {
|
||||
s += " (" + n.MultiNode + ")"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// enginesColumn is each engine with its version: llamacpp b10991.
|
||||
func enginesColumn(n adminapi.NodeView) string {
|
||||
if len(n.EngineVersions) == 0 {
|
||||
return join(n.Engines)
|
||||
}
|
||||
parts := make([]string, 0, len(n.Engines))
|
||||
for _, e := range n.Engines {
|
||||
if v := n.EngineVersions[e]; v != "" && v != "unknown" {
|
||||
e += " " + v
|
||||
}
|
||||
parts = append(parts, e)
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
// classesColumn is the node's GPU classes, with their compute capability
|
||||
// when they report one: 6000-24g (7.5).
|
||||
func classesColumn(n adminapi.NodeView) string {
|
||||
s := classesOnly(n)
|
||||
if len(n.ComputeCapabilities) > 0 && s != "-" {
|
||||
s += " (" + strings.Join(n.ComputeCapabilities, ",") + ")"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func classesOnly(n adminapi.NodeView) string {
|
||||
counted := len(n.ClassGPUs) > 1
|
||||
for _, c := range n.ClassGPUs {
|
||||
counted = counted || c.MIG
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// newMeshJobsCmd is the 3D models clients asked for, as jobs: every owner's,
|
||||
// what each is doing, and a way to stop one. (ctl jobs is the control
|
||||
// plane's own work, serving and updating models.)
|
||||
func newMeshJobsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "3d",
|
||||
Short: "3D models clients asked for: the jobs, how far along each is, and stopping one",
|
||||
}
|
||||
var status string
|
||||
var limit int
|
||||
list := &cobra.Command{
|
||||
Use: "list",
|
||||
Short: "Every owner's 3D jobs, newest first",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
q := url.Values{}
|
||||
if status != "" {
|
||||
q.Set("status", status)
|
||||
}
|
||||
if limit > 0 {
|
||||
q.Set("limit", fmt.Sprint(limit))
|
||||
}
|
||||
var jobs []adminapi.JobView
|
||||
if err := client().get(cmd.Context(), "/api/v1/client-jobs?"+q.Encode(), &jobs); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return emitJSON(jobs)
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "no 3D jobs")
|
||||
return nil
|
||||
}
|
||||
w := tabTo(cmd.OutOrStdout())
|
||||
fmt.Fprintln(w, "JOB\tMODEL\tSTATUS\tPROGRESS\tOWNER\tASKED")
|
||||
for _, j := range jobs {
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s ago\n", j.ID, j.Model, j.Status, progressText(j),
|
||||
j.Owner, time.Since(j.CreatedAt).Round(time.Second))
|
||||
}
|
||||
return w.Flush()
|
||||
},
|
||||
}
|
||||
list.Flags().StringVar(&status, "status", "", "only jobs in this status: queued, running, completed, failed…")
|
||||
list.Flags().IntVar(&limit, "limit", 0, "the most jobs to show (50)")
|
||||
get := &cobra.Command{
|
||||
Use: "get JOB",
|
||||
Short: "One 3D job: its status, progress, result or error",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var j adminapi.JobView
|
||||
if err := client().get(cmd.Context(), "/api/v1/client-jobs/"+url.PathEscape(args[0]), &j); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return emitJSON(j)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "job %s\nmodel %s\nstatus %s\n", j.ID, j.Model, j.Status)
|
||||
if p := progressText(j); p != "" {
|
||||
fmt.Fprintf(out, "progress %s\n", p)
|
||||
}
|
||||
fmt.Fprintf(out, "owner %s (key %s)\nasked %s\n", j.Owner, j.KeyID, j.CreatedAt.UTC().Format(time.RFC3339))
|
||||
if j.FinishedAt != nil && j.StartedAt != nil {
|
||||
fmt.Fprintf(out, "took %s\n", j.FinishedAt.Sub(*j.StartedAt).Round(time.Second))
|
||||
}
|
||||
if j.OutputFileID != "" {
|
||||
fmt.Fprintf(out, "result %s (%s)\n", j.OutputFileID, j.ContentType)
|
||||
}
|
||||
if j.Error != "" {
|
||||
fmt.Fprintf(out, "error %s\n", j.Error)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cancel := &cobra.Command{
|
||||
Use: "cancel JOB",
|
||||
Short: "Stop a 3D job, in the engine's queue too",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var j adminapi.JobView
|
||||
if err := client().post(cmd.Context(), "/api/v1/client-jobs/"+url.PathEscape(args[0])+"/cancel", nil, &j); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s is %s\n", j.ID, j.Status)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.AddCommand(list, get, cancel)
|
||||
return cmd
|
||||
}
|
||||
|
||||
// progressText is a job's progress as a person reads it.
|
||||
func progressText(j adminapi.JobView) string {
|
||||
if len(j.Progress) == 0 {
|
||||
return ""
|
||||
}
|
||||
var p struct {
|
||||
State string `json:"state"`
|
||||
Ahead int `json:"ahead"`
|
||||
NodeClass string `json:"node_class"`
|
||||
Step int `json:"step"`
|
||||
Steps int `json:"steps"`
|
||||
Percent int `json:"percent"`
|
||||
}
|
||||
if json.Unmarshal(j.Progress, &p) != nil {
|
||||
return ""
|
||||
}
|
||||
switch {
|
||||
case p.State == "queued" && p.Ahead > 0:
|
||||
return fmt.Sprintf("queued, %d ahead", p.Ahead)
|
||||
case p.State == "running" && p.Steps > 0:
|
||||
return fmt.Sprintf("running %d%%, %s step %d/%d", p.Percent, p.NodeClass, p.Step, p.Steps)
|
||||
case p.State == "running" && p.NodeClass != "":
|
||||
return "running " + p.NodeClass
|
||||
}
|
||||
return p.State
|
||||
}
|
||||
@@ -56,6 +56,12 @@ func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
fmt.Fprintf(out, "status %d\n", v.Status)
|
||||
fmt.Fprintf(out, "time %d ms total: %d queued, %d to the first token\n", v.LatencyMs, v.QueueMs, v.TTFTMs)
|
||||
fmt.Fprintf(out, "tokens %d in, %d out\n", v.PromptTokens, v.CompletionTokens)
|
||||
if v.Images > 0 {
|
||||
fmt.Fprintf(out, "images %d\n", v.Images)
|
||||
}
|
||||
if v.Meshes > 0 {
|
||||
fmt.Fprintf(out, "meshes %d\n", v.Meshes)
|
||||
}
|
||||
if v.CacheHit {
|
||||
fmt.Fprintf(out, "cache answered from the project's answer cache: nothing generated, nothing charged\n")
|
||||
}
|
||||
|
||||
@@ -12,9 +12,10 @@ import (
|
||||
|
||||
func newRestartCmd(client func() *ctlClient) *cobra.Command {
|
||||
var (
|
||||
models []string
|
||||
all bool
|
||||
inPlace bool
|
||||
models []string
|
||||
all bool
|
||||
inPlace bool
|
||||
surge, unavailable int
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "restart [REPLICA...]",
|
||||
@@ -31,6 +32,10 @@ func newRestartCmd(client func() *ctlClient) *cobra.Command {
|
||||
"profile edited to need more than any node has — the restart is given up\n" +
|
||||
"and the old replica keeps serving: llmgw ctl replicas says so, and why.\n\n" +
|
||||
"--in-place stops each replica first, whatever room there is.\n\n" +
|
||||
"A model's replicas are replaced one at a time. --max-surge N replaces N at\n" +
|
||||
"once, each beside itself, needing room for N more; --max-unavailable N\n" +
|
||||
"stops N at once first, needing no room and leaving N fewer serving while\n" +
|
||||
"they load. Together, the first N are stopped first and the rest surged.\n\n" +
|
||||
" llmgw ctl restart qwen2.5-7b-3f2a91c0\n" +
|
||||
" llmgw ctl restart --model qwen2.5-7b\n" +
|
||||
" llmgw ctl restart --all",
|
||||
@@ -38,7 +43,8 @@ func newRestartCmd(client func() *ctlClient) *cobra.Command {
|
||||
if len(args) == 0 && len(models) == 0 && !all {
|
||||
return errors.New("name replicas, --model, or --all")
|
||||
}
|
||||
body, _ := json.Marshal(adminapi.RestartRequest{Replicas: args, Models: models, All: all, InPlace: inPlace})
|
||||
body, _ := json.Marshal(adminapi.RestartRequest{Replicas: args, Models: models, All: all, InPlace: inPlace,
|
||||
MaxSurge: surge, MaxUnavailable: unavailable})
|
||||
var res adminapi.RestartResult
|
||||
if err := client().post(cmd.Context(), "/api/v1/restart", body, &res); err != nil {
|
||||
return err
|
||||
@@ -50,6 +56,9 @@ func newRestartCmd(client func() *ctlClient) *cobra.Command {
|
||||
c.Flags().StringArrayVar(&models, "model", nil, "every replica of this model; repeatable")
|
||||
c.Flags().BoolVar(&all, "all", false, "every ready replica in the cluster")
|
||||
c.Flags().BoolVar(&inPlace, "in-place", false, "stop each replica before starting its replacement")
|
||||
c.Flags().IntVar(&surge, "max-surge", 0, "replicas of a model replaced beside themselves at once (default 1)")
|
||||
c.Flags().IntVar(&unavailable, "max-unavailable", 0,
|
||||
"replicas of a model stopped first at once, each out while its replacement loads")
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
@@ -268,6 +268,26 @@ func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient,
|
||||
}
|
||||
fmt.Fprintf(out, "%d node(s) upgraded to %s in %s, %d already on it\n",
|
||||
done, v, time.Since(start).Round(time.Second), skipped)
|
||||
// Checked at the end, against what the control plane says now: a node
|
||||
// that is not on the version is said by name, and the command fails. A
|
||||
// site's nodes stayed a release behind with nothing printed to say so.
|
||||
if len(nodes) == 0 {
|
||||
fmt.Fprintln(out, "This control plane has no nodes registered, so none were upgraded.")
|
||||
}
|
||||
var now []adminapi.NodeView
|
||||
if err := client().get(ctx, "/api/v1/nodes", &now); err == nil {
|
||||
var behind []string
|
||||
for _, n := range now {
|
||||
if (len(only) == 0 || slices.Contains(only, n.ID)) && n.AgentVersion != v {
|
||||
behind = append(behind, fmt.Sprintf("%s (%s, %s)", n.ID, orUnknown(n.AgentVersion), n.State))
|
||||
}
|
||||
}
|
||||
if len(behind) > 0 {
|
||||
slices.Sort(behind)
|
||||
return fmt.Errorf("not on %s: %s\nrun this again to try them, or `llmgw ctl logs NODE` for why",
|
||||
v, strings.Join(behind, ", "))
|
||||
}
|
||||
}
|
||||
// The one machine this does not reach. Said every time, because it is
|
||||
// exactly the kind of thing that stays three versions behind unnoticed.
|
||||
fmt.Fprintf(out, "The model mirror's host is not upgraded by this; on it, run:\n"+
|
||||
|
||||
@@ -150,3 +150,54 @@ func TestABinaryInstalledByHandIsNotAServerRunningIt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpgradeThatFindsNoNodesSaysSo(t *testing.T) {
|
||||
// A site's nodes stayed a release behind with nothing printed to say
|
||||
// that none had been reached.
|
||||
f := &fakeFleet{versions: map[string]string{}}
|
||||
srv := f.server(t)
|
||||
var out bytes.Buffer
|
||||
if err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, time.Second, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "no nodes registered") {
|
||||
t.Errorf("output:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpgradeFailsNamingANodeLeftBehind(t *testing.T) {
|
||||
// A node that joins, or comes back on its old binary, while the waves
|
||||
// run is found at the end rather than passed over.
|
||||
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0"}}
|
||||
srv := f.server(t)
|
||||
late := func() *ctlClient {
|
||||
f.mu.Lock()
|
||||
if len(f.told) == 1 {
|
||||
f.versions["late"] = "v1.0.0"
|
||||
}
|
||||
f.mu.Unlock()
|
||||
return newCtlClient(srv.URL)
|
||||
}
|
||||
var out bytes.Buffer
|
||||
err := upgradeNodes(t.Context(), &out, late, "v1.0.1", nil, 5*time.Second, 1)
|
||||
if err == nil || !strings.Contains(err.Error(), "late (v1.0.0, ready)") {
|
||||
t.Errorf("err %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpgradingAControlPlaneAloneSaysItsNodesWereNot(t *testing.T) {
|
||||
f := &fakeFleet{versions: map[string]string{"p-g1": "v1.0.0", "p-g3": "v1.0.1"}}
|
||||
srv := f.server(t)
|
||||
var out bytes.Buffer
|
||||
nodesLeftBehind(t.Context(), &out, newCtlClient(srv.URL), "v1.0.1")
|
||||
if !strings.Contains(out.String(), "1 of 2 node(s) are not on v1.0.1") ||
|
||||
!strings.Contains(out.String(), "ctl upgrade v1.0.1 --skip-control-plane") {
|
||||
t.Errorf("output:\n%s", out.String())
|
||||
}
|
||||
out.Reset()
|
||||
f.versions["p-g1"] = "v1.0.1"
|
||||
nodesLeftBehind(t.Context(), &out, newCtlClient(srv.URL), "v1.0.1")
|
||||
if out.Len() != 0 {
|
||||
t.Errorf("said something with every node on the version: %s", out.String())
|
||||
}
|
||||
}
|
||||
+22
-7
@@ -63,9 +63,10 @@ that file when the report is not enough to go on.`,
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(struct {
|
||||
Findings []doctor.Finding `json:"findings"`
|
||||
Facts doctor.Facts `json:"facts"`
|
||||
}{findings, facts})
|
||||
Findings []doctor.Finding `json:"findings"`
|
||||
Readiness doctor.Readiness `json:"readiness"`
|
||||
Facts doctor.Facts `json:"facts"`
|
||||
}{findings, doctor.ReadinessOf(findings), facts})
|
||||
}
|
||||
printDoctor(os.Stdout, facts, findings, verbose)
|
||||
for _, f := range findings {
|
||||
@@ -101,12 +102,26 @@ func printDoctor(out io.Writer, f doctor.Facts, findings []doctor.Finding, verbo
|
||||
fmt.Fprintf(out, "%s · %s %s %s · driver %s\n\n", f.Hostname, f.Host.OSID, f.Host.OSVersion,
|
||||
f.Host.Arch, orDash(f.Host.NVIDIADriver))
|
||||
|
||||
for _, x := range findings {
|
||||
fmt.Fprintf(out, "%-4s %-6s %s\n", x.Level, x.Area, x.Message)
|
||||
if x.Fix != "" && (x.Level == doctor.Fail || x.Level == doctor.Warn) {
|
||||
fmt.Fprintf(out, " → %s\n", x.Fix)
|
||||
// The node first; then what replicas spanning nodes would need, which
|
||||
// keeps a node out only when a fabric was declared for it.
|
||||
multi := false
|
||||
for _, pass := range []bool{false, true} {
|
||||
for _, x := range findings {
|
||||
if x.MultiNode() != pass {
|
||||
continue
|
||||
}
|
||||
if pass && !multi {
|
||||
fmt.Fprintln(out, "\nMulti-node (fabrics, RDMA) — for replicas spanning nodes only:")
|
||||
multi = true
|
||||
}
|
||||
fmt.Fprintf(out, "%-4s %-6s %s\n", x.Level, x.Area, x.Message)
|
||||
if x.Fix != "" && (x.Level == doctor.Fail || x.Level == doctor.Warn) {
|
||||
fmt.Fprintf(out, " → %s\n", x.Fix)
|
||||
}
|
||||
}
|
||||
}
|
||||
r := doctor.ReadinessOf(findings)
|
||||
fmt.Fprintf(out, "\nThis node: %s\nMulti-node: %s\n", r.Node, r.MultiNode)
|
||||
|
||||
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
|
||||
if len(f.GPUs) > 0 {
|
||||
|
||||
+166
-1
@@ -4,10 +4,15 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
@@ -22,6 +27,7 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
nodes int
|
||||
tp int
|
||||
perNode int
|
||||
model string
|
||||
)
|
||||
|
||||
cmd := &cobra.Command{
|
||||
@@ -31,6 +37,10 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
"engines rename and remove flags between versions. When that happens the\n" +
|
||||
"failure arrives minutes into a model load, in an engine's own error\n" +
|
||||
"message, on a node nobody was watching.\n\n" +
|
||||
"For llama.cpp on a machine with NVIDIA GPUs it then loads a small GGUF\n" +
|
||||
"(--model, or the smallest in the weights cache) with every layer on the GPU\n" +
|
||||
"and asks for one token: a build whose CUDA backend does not load computes on\n" +
|
||||
"the CPU, and that is where it is caught.\n\n" +
|
||||
"This asks the engine itself. It runs the engine's --help and checks every\n" +
|
||||
"flag llmgw would pass against it, which needs no GPU — a CLI prints its\n" +
|
||||
"help on any machine. Run it after upgrading an engine.\n\n" +
|
||||
@@ -49,6 +59,11 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
// rarely on PATH: a check that cannot find the engine reports
|
||||
// nothing rather than reporting a problem.
|
||||
cfg := engines.Config{}
|
||||
// The build the agent installed, when none is named: the newest
|
||||
// under /opt/llmgw/engines, which is where every node keeps it.
|
||||
if bin == "" && name == "llamacpp" {
|
||||
bin = installedLlama()
|
||||
}
|
||||
if bin != "" {
|
||||
switch name {
|
||||
case "llamacpp":
|
||||
@@ -135,6 +150,11 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
var deviceErr error
|
||||
if fb, ok := eng.(engines.CPUFallback); ok && help == "" {
|
||||
deviceErr = checkDevices(cmd.Context(), name, fb, out)
|
||||
// Seeing a GPU is not computing on it: a small model loaded
|
||||
// on it and asked for a token is.
|
||||
if b, ok := eng.(interface{ Binary() string }); ok && deviceErr == nil && model != "none" {
|
||||
deviceErr = computeOnGPU(cmd.Context(), b.Binary(), model, out)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintf(out, "%s: %d flags checked", name, checked)
|
||||
@@ -160,7 +180,9 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
|
||||
f := cmd.Flags()
|
||||
f.StringVar(&help, "help-output", "", "a saved --help to check against, instead of running the engine")
|
||||
f.StringVar(&bin, "bin", "", "where the engine lives, when it is not on PATH")
|
||||
f.StringVar(&bin, "bin", "", "where the engine lives (default: the build the agent installed, else PATH)")
|
||||
f.StringVar(&model, "model", "", "a GGUF file to load on the GPU and ask for a token (default: the smallest "+
|
||||
"in this node's weights cache; none skips it)")
|
||||
f.IntVar(&nodes, "nodes", 2, "shape the command for a replica of this many nodes")
|
||||
f.IntVar(&tp, "tp", 8, "tensor parallel degree")
|
||||
f.IntVar(&perNode, "gpus-per-node", 4, "GPUs each node contributes")
|
||||
@@ -286,3 +308,146 @@ func checkDevices(ctx context.Context, name string, fb engines.CPUFallback, out
|
||||
fmt.Fprintf(out, "computes on: %s\n", strings.Join(devs, "; "))
|
||||
return nil
|
||||
}
|
||||
|
||||
// llamaRoot is where the agent installs llama.cpp builds.
|
||||
var llamaRoot = "/opt/llmgw/engines/llamacpp"
|
||||
|
||||
// installedLlama is the llama-server the agent installed here: under
|
||||
// llamaRoot/<version-variant>/llama-<build>/, or one level up for older
|
||||
// layouts; on a machine with NVIDIA GPUs a CUDA build first, then the newest.
|
||||
func installedLlama() string {
|
||||
var found []string
|
||||
for _, pat := range []string{"*/llama-server", "*/*/llama-server"} {
|
||||
m, _ := filepath.Glob(filepath.Join(llamaRoot, pat))
|
||||
found = append(found, m...)
|
||||
}
|
||||
if len(found) == 0 {
|
||||
return ""
|
||||
}
|
||||
sort.Strings(found)
|
||||
if _, err := exec.LookPath("nvidia-smi"); err == nil {
|
||||
for i := len(found) - 1; i >= 0; i-- {
|
||||
if strings.Contains(found[i], "cuda") {
|
||||
return found[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
return found[len(found)-1]
|
||||
}
|
||||
|
||||
// weightDirs are where a node keeps models it has fetched.
|
||||
var weightDirs = []string{"/var/lib/llmgw-weights", "/var/lib/llmgw-models", "/srv/models"}
|
||||
|
||||
// smallestGGUF is the smallest GGUF file under dirs, for a check that only
|
||||
// needs something to load.
|
||||
func smallestGGUF(dirs []string) string {
|
||||
best, size := "", int64(0)
|
||||
for _, d := range dirs {
|
||||
_ = filepath.WalkDir(d, func(p string, e os.DirEntry, err error) error {
|
||||
if err != nil || e.IsDir() || !strings.HasSuffix(strings.ToLower(p), ".gguf") ||
|
||||
strings.Contains(strings.ToLower(filepath.Base(p)), "mmproj") {
|
||||
return nil
|
||||
}
|
||||
if info, err := e.Info(); err == nil && (best == "" || info.Size() < size) {
|
||||
best, size = p, info.Size()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// computeOnGPU loads a model with every layer on the GPU, asks for one
|
||||
// token, and reads from llama.cpp's own output where the layers went.
|
||||
func computeOnGPU(ctx context.Context, bin, model string, out io.Writer) error {
|
||||
if _, err := exec.LookPath("nvidia-smi"); err != nil {
|
||||
return nil
|
||||
}
|
||||
if model == "" {
|
||||
if model = smallestGGUF(weightDirs); model == "" {
|
||||
fmt.Fprintln(out, "no GGUF in this node's weights cache to compute with; --model PATH runs one")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
port := ln.Addr().(*net.TCPAddr).Port
|
||||
_ = ln.Close()
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
|
||||
defer cancel()
|
||||
var logs syncBuffer
|
||||
c := exec.CommandContext(ctx, bin, "-m", model, "--host", "127.0.0.1", "--port", strconv.Itoa(port), //nolint:gosec // the engine's own binary
|
||||
"--n-gpu-layers", "999", "--ctx-size", "512", "--parallel", "1",
|
||||
// Where the layers went is said only at this verbosity.
|
||||
"-lv", "4")
|
||||
c.Stdout, c.Stderr = &logs, &logs
|
||||
if err := c.Start(); err != nil {
|
||||
return fmt.Errorf("starting %s: %w", bin, err)
|
||||
}
|
||||
defer func() { _ = c.Process.Kill(); _ = c.Wait() }()
|
||||
base := "http://127.0.0.1:" + strconv.Itoa(port)
|
||||
start := time.Now()
|
||||
for {
|
||||
resp, err := http.Get(base + "/health") //nolint:gosec,noctx // loopback, bounded by the loop
|
||||
if err == nil {
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return fmt.Errorf("%s did not come up with %s: %s", bin, filepath.Base(model), tailLines(logs.String(), 5))
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, base+"/completion",
|
||||
strings.NewReader(`{"prompt":"Hello","n_predict":1}`))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("asking it for a token: %w", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
took := time.Since(start)
|
||||
comp, said := engines.ReadLlamaCompute(strings.Split(logs.String(), "\n"))
|
||||
if !said {
|
||||
fmt.Fprintf(out, "computed a token with %s, but llama.cpp did not say where\n", filepath.Base(model))
|
||||
return nil
|
||||
}
|
||||
if comp.OnTheCPU() {
|
||||
return fmt.Errorf("%s computed on the CPU though every layer was asked for on the GPU (%d of %d offloaded): "+
|
||||
"its CUDA backend did not load; ldd on its libggml-cuda.so names the missing library",
|
||||
filepath.Base(model), comp.Offloaded, comp.Total)
|
||||
}
|
||||
fmt.Fprintf(out, "computed a token on %s: %s, %d of %d layers offloaded, in %s\n", comp.Backend,
|
||||
strings.Join(comp.Devices, "; "), comp.Offloaded, comp.Total, took.Round(100*time.Millisecond))
|
||||
return nil
|
||||
}
|
||||
|
||||
func tailLines(s string, n int) string {
|
||||
l := strings.Split(strings.TrimSpace(s), "\n")
|
||||
if len(l) > n {
|
||||
l = l[len(l)-n:]
|
||||
}
|
||||
return strings.Join(l, " | ")
|
||||
}
|
||||
|
||||
// syncBuffer is a buffer two writers share.
|
||||
type syncBuffer struct {
|
||||
mu sync.Mutex
|
||||
b strings.Builder
|
||||
}
|
||||
|
||||
func (s *syncBuffer) Write(p []byte) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.b.Write(p)
|
||||
}
|
||||
|
||||
func (s *syncBuffer) String() string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.b.String()
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestMain lets the test binary stand in for llama-server: run with
|
||||
// LLMGW_FAKE_LLAMA set, it prints what llama.cpp prints and answers.
|
||||
func TestMain(m *testing.M) {
|
||||
if mode := os.Getenv("LLMGW_FAKE_LLAMA"); mode != "" {
|
||||
fakeLlama(mode)
|
||||
return
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func fakeLlama(mode string) {
|
||||
port := ""
|
||||
for i, a := range os.Args {
|
||||
if a == "--port" && i+1 < len(os.Args) {
|
||||
port = os.Args[i+1]
|
||||
}
|
||||
}
|
||||
if mode == "gpu" {
|
||||
fmt.Println("llama_model_load_from_file_impl: using device CUDA0 (Quadro RTX 6000) - 23800 MiB free")
|
||||
fmt.Println("load_tensors: offloaded 25/25 layers to GPU")
|
||||
} else {
|
||||
fmt.Println("warning: no usable GPU found, --gpu-layers option will be ignored")
|
||||
fmt.Println("load_tensors: offloaded 0/25 layers to GPU")
|
||||
}
|
||||
http.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })
|
||||
http.HandleFunc("/completion", func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{"content":"!"}`)) })
|
||||
_ = http.ListenAndServe("127.0.0.1:"+port, nil) //nolint:gosec // a test's stand-in
|
||||
}
|
||||
|
||||
func TestEngineCheckComputesATokenOnTheGPU(t *testing.T) {
|
||||
// nvidia-smi present, so the check runs.
|
||||
dir := t.TempDir()
|
||||
smi := filepath.Join(dir, "nvidia-smi")
|
||||
if err := os.WriteFile(smi, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
model := filepath.Join(dir, "tiny.gguf")
|
||||
_ = os.WriteFile(model, []byte("GGUF"), 0o644)
|
||||
self, _ := os.Executable()
|
||||
|
||||
t.Setenv("LLMGW_FAKE_LLAMA", "gpu")
|
||||
var out bytes.Buffer
|
||||
if err := computeOnGPU(t.Context(), self, model, &out); err != nil || !strings.Contains(out.String(), "on CUDA") ||
|
||||
!strings.Contains(out.String(), "25 of 25") {
|
||||
t.Fatalf("on the GPU: %v %s", err, out.String())
|
||||
}
|
||||
|
||||
t.Setenv("LLMGW_FAKE_LLAMA", "cpu")
|
||||
out.Reset()
|
||||
if err := computeOnGPU(t.Context(), self, model, &out); err == nil || !strings.Contains(err.Error(), "computed on the CPU") {
|
||||
t.Fatalf("a build that falls back to the CPU: %v %s", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineCheckFindsTheBuildTheAgentInstalled(t *testing.T) {
|
||||
old := llamaRoot
|
||||
llamaRoot = t.TempDir()
|
||||
t.Cleanup(func() { llamaRoot = old })
|
||||
if got := installedLlama(); got != "" {
|
||||
t.Fatalf("nothing installed, found %q", got)
|
||||
}
|
||||
for _, p := range []string{"b10991-cpu/llama-b10991/llama-server", "b10991-cuda-13.3/llama-b10991/llama-server"} {
|
||||
full := filepath.Join(llamaRoot, p)
|
||||
_ = os.MkdirAll(filepath.Dir(full), 0o755)
|
||||
_ = os.WriteFile(full, []byte("x"), 0o755)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
_ = os.WriteFile(filepath.Join(dir, "nvidia-smi"), []byte("#!/bin/sh\n"), 0o755)
|
||||
t.Setenv("PATH", dir)
|
||||
if got := installedLlama(); !strings.Contains(got, "cuda-13.3/llama-b10991/llama-server") {
|
||||
t.Errorf("with GPUs, not the CUDA build: %q", got)
|
||||
}
|
||||
}
|
||||
+49
-21
@@ -7,12 +7,14 @@ import (
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
)
|
||||
|
||||
// firewallStateFile is where a control plane writes what the cluster's
|
||||
@@ -20,11 +22,9 @@ import (
|
||||
// the server itself runs as llmgw and cannot change the ruleset.
|
||||
const firewallStateFile = "/var/lib/llmgw/firewall.json"
|
||||
|
||||
// firewallState is what is written there.
|
||||
type firewallState struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Networks []string `json:"networks,omitempty"`
|
||||
}
|
||||
// firewallState is what is written there: the same as the mirror is told
|
||||
// and keeps beside its models.
|
||||
type firewallState = mirror.FirewallState
|
||||
|
||||
// writeFirewallState records the settings' firewall for this machine's sync
|
||||
// unit. Without the state directory — a server not run by systemd — there
|
||||
@@ -46,25 +46,34 @@ func writeFirewallState(cfg config.Server) error {
|
||||
|
||||
// The units that apply it: a path unit watching the file, and the oneshot it
|
||||
// starts.
|
||||
const (
|
||||
firewallSyncPathUnit = "llmgw-firewall-sync.path"
|
||||
firewallSyncUnitFile = `[Unit]
|
||||
Description=llmgw firewall: apply the cluster's settings on this control plane
|
||||
const firewallSyncPathUnit = "llmgw-firewall-sync.path"
|
||||
|
||||
// firewallSyncUnits are the two units for a state file: on a control plane
|
||||
// the one its server writes, on a mirror the one the control plane has the
|
||||
// mirror write.
|
||||
func firewallSyncUnits(state string) (service, pathUnit string) {
|
||||
arg := ""
|
||||
if state != firewallStateFile {
|
||||
arg = " --state " + state
|
||||
}
|
||||
service = `[Unit]
|
||||
Description=llmgw firewall: apply the cluster's settings on this machine
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/llmgw firewall sync
|
||||
ExecStart=/usr/local/bin/llmgw firewall sync` + arg + `
|
||||
`
|
||||
firewallSyncPathFile = `[Unit]
|
||||
Description=llmgw firewall: follow the cluster's settings on this control plane
|
||||
pathUnit = `[Unit]
|
||||
Description=llmgw firewall: follow the cluster's settings on this machine
|
||||
|
||||
[Path]
|
||||
PathChanged=` + firewallStateFile + `
|
||||
PathChanged=` + state + `
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`
|
||||
)
|
||||
return service, pathUnit
|
||||
}
|
||||
|
||||
func newFirewallCmd() *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
@@ -76,6 +85,7 @@ func newFirewallCmd() *cobra.Command {
|
||||
"which " + firewallSyncPathUnit + " applies with 'llmgw firewall sync'.",
|
||||
}
|
||||
var install bool
|
||||
state := firewallStateFile
|
||||
sync := &cobra.Command{
|
||||
Use: "sync",
|
||||
Short: "Apply what the cluster's settings say of the firewall, here (root)",
|
||||
@@ -85,21 +95,21 @@ func newFirewallCmd() *cobra.Command {
|
||||
return errors.New("the firewall is changed as root: run it with sudo")
|
||||
}
|
||||
if install {
|
||||
if err := installFirewallSync(cmd.Context(), firewall.Local{}); err != nil {
|
||||
if err := installFirewallSync(cmd.Context(), firewall.Local{}, state); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(cmd.OutOrStdout(), firewallSyncPathUnit+" follows the cluster's settings here")
|
||||
if _, err := os.Stat(firewallStateFile); err != nil {
|
||||
return nil // the server writes it when it starts
|
||||
if _, err := os.Stat(state); err != nil {
|
||||
return nil // written when the settings are first told
|
||||
}
|
||||
}
|
||||
b, err := os.ReadFile(firewallStateFile)
|
||||
b, err := os.ReadFile(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var st firewallState
|
||||
if err := json.Unmarshal(b, &st); err != nil {
|
||||
return fmt.Errorf("%s: %w", firewallStateFile, err)
|
||||
return fmt.Errorf("%s: %w", state, err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 2*time.Minute)
|
||||
defer cancel()
|
||||
@@ -112,14 +122,17 @@ func newFirewallCmd() *cobra.Command {
|
||||
},
|
||||
}
|
||||
sync.Flags().BoolVar(&install, "install", false, "first install the units that run this whenever the settings change")
|
||||
sync.Flags().StringVar(&state, "state", firewallStateFile,
|
||||
"the file the settings are written to: a control plane's, or "+mirror.FirewallFile+" in a mirror's directory")
|
||||
c.AddCommand(sync)
|
||||
return c
|
||||
}
|
||||
|
||||
// installFirewallSync writes the sync units on a machine and has the path
|
||||
// unit follow the state file.
|
||||
func installFirewallSync(ctx context.Context, r firewall.Runner) error {
|
||||
body := firewallSyncUnitFile + "\x00" + firewallSyncPathFile
|
||||
func installFirewallSync(ctx context.Context, r firewall.Runner, state string) error {
|
||||
service, pathUnit := firewallSyncUnits(state)
|
||||
body := service + "\x00" + pathUnit
|
||||
out, err := r.Run(ctx, `set -e
|
||||
t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
|
||||
cat > "$t/all"
|
||||
@@ -172,5 +185,20 @@ func syncFirewall(ctx context.Context, r firewall.Runner, st firewallState) (str
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
// The cluster's machines: a node on a network this one has not let in
|
||||
// yet, which the mirror's files are for.
|
||||
if len(st.Toward) > 0 {
|
||||
added, err := firewall.Admit(ctx, r, st.Toward)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(added) > 0 {
|
||||
shown := make([]string, len(added))
|
||||
for i, p := range added {
|
||||
shown[i] = p.String()
|
||||
}
|
||||
return "llmgw's firewall is on here, and now lets in " + strings.Join(shown, ", "), nil
|
||||
}
|
||||
}
|
||||
return "llmgw's firewall is on here", nil
|
||||
}
|
||||
@@ -2,8 +2,11 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
|
||||
)
|
||||
|
||||
type scriptLog struct {
|
||||
@@ -49,3 +52,32 @@ func TestAControlPlaneFollowsTheFirewallSetting(t *testing.T) {
|
||||
t.Errorf("on: %q %v", msg, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMachineLetsInTheClusterMachinesItIsTold(t *testing.T) {
|
||||
// The rules hold 10.0.0.0/24; a node on 192.168.7.0/24 is let in, as
|
||||
// the route to it says, and one already in is not asked about.
|
||||
r := &scriptLog{reply: func(s string) string {
|
||||
switch {
|
||||
case strings.Contains(s, "echo loaded"):
|
||||
return "loaded\n"
|
||||
case strings.Contains(s, "cat /etc/llmgw/firewall.nft"):
|
||||
return firewallRulesFor("10.0.0.0/24")
|
||||
case strings.Contains(s, "ip -o route get 192.168.7.20"):
|
||||
return "192.168.7.20 dev eth1 src 192.168.7.1 uid 0\n2: eth1 inet 192.168.7.1/24 brd 192.168.7.255 scope global eth1\n"
|
||||
case strings.Contains(s, "nft -f"):
|
||||
return "changed\n"
|
||||
}
|
||||
return ""
|
||||
}}
|
||||
msg, err := syncFirewall(context.Background(), r, firewallState{Enabled: true, Toward: []string{"10.0.0.147", "192.168.7.20"}})
|
||||
if err != nil || !strings.Contains(msg, "192.168.7.0/24") {
|
||||
t.Fatalf("%q %v", msg, err)
|
||||
}
|
||||
if strings.Contains(strings.Join(r.scripts, "\n"), "route get 10.0.0.147") {
|
||||
t.Error("asked the route to a machine already let in")
|
||||
}
|
||||
}
|
||||
|
||||
func firewallRulesFor(nets string) string {
|
||||
return firewall.Script(firewall.Spec{Ports: []firewall.Port{firewall.MirrorFiles}, Allow: []netip.Prefix{netip.MustParsePrefix(nets)}})
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// firewallStateKey is where, in the store, the firewall's state is kept
|
||||
// for the store's members to read.
|
||||
const firewallStateKey = "firewall"
|
||||
|
||||
// firewallPublisher tells the machines that cannot ask a control plane
|
||||
// what the cluster's settings say of their firewall: the mirror, over its
|
||||
// API, and the store's members, through the store they hold a copy of.
|
||||
// Nodes are told over their agents' streams and control planes by their
|
||||
// own servers. On the leader only, at start, at every change to the
|
||||
// firewall's settings, and every few minutes for nodes come and gone.
|
||||
type firewallPublisher struct {
|
||||
settings func() config.Server
|
||||
st *store.DB
|
||||
nodes func() []string
|
||||
leader func() bool
|
||||
log *slog.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
mirror *mirror.Client
|
||||
kick chan struct{}
|
||||
}
|
||||
|
||||
func newFirewallPublisher(settings func() config.Server, st *store.DB, nodes func() []string,
|
||||
leader func() bool, log *slog.Logger) *firewallPublisher {
|
||||
return &firewallPublisher{settings: settings, st: st, nodes: nodes, leader: leader, log: log,
|
||||
kick: make(chan struct{}, 1)}
|
||||
}
|
||||
|
||||
// setMirror has the mirror told too.
|
||||
func (p *firewallPublisher) setMirror(m *mirror.Client) {
|
||||
p.mu.Lock()
|
||||
p.mirror = m
|
||||
p.mu.Unlock()
|
||||
p.Kick()
|
||||
}
|
||||
|
||||
// Kick has the state worked out and told again now.
|
||||
func (p *firewallPublisher) Kick() {
|
||||
select {
|
||||
case p.kick <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (p *firewallPublisher) run(ctx context.Context, every time.Duration) {
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
p.once(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
case <-p.kick:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// state is what the settings say now, with the cluster's machines.
|
||||
func (p *firewallPublisher) state(ctx context.Context) mirror.FirewallState {
|
||||
cfg := p.settings()
|
||||
st := mirror.FirewallState{Enabled: cfg.Firewall.Enabled, Networks: slices.Clone(cfg.FirewallNetworks)}
|
||||
var toward []string
|
||||
if p.nodes != nil {
|
||||
toward = append(toward, p.nodes()...)
|
||||
}
|
||||
if p.st != nil {
|
||||
if cps, err := p.st.ControlPlanes(ctx); err == nil {
|
||||
for _, c := range cps {
|
||||
toward = append(toward, c.URL)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, t := range toward {
|
||||
if a, ok := hostAddr(t); ok {
|
||||
st.Toward = append(st.Toward, a)
|
||||
}
|
||||
}
|
||||
slices.Sort(st.Toward)
|
||||
st.Toward = slices.Compact(st.Toward)
|
||||
return st
|
||||
}
|
||||
|
||||
// hostAddr is the address in an address, host:port or URL; names are not
|
||||
// looked up here.
|
||||
func hostAddr(s string) (string, bool) {
|
||||
if u, err := url.Parse(s); err == nil && u.Host != "" {
|
||||
s = u.Hostname()
|
||||
} else if h, _, err := net.SplitHostPort(s); err == nil {
|
||||
s = h
|
||||
}
|
||||
a, err := netip.ParseAddr(s)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return a.Unmap().String(), true
|
||||
}
|
||||
|
||||
func (p *firewallPublisher) once(ctx context.Context) {
|
||||
if p.leader != nil && !p.leader() {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
st := p.state(ctx)
|
||||
p.mu.Lock()
|
||||
m := p.mirror
|
||||
p.mu.Unlock()
|
||||
if m != nil {
|
||||
changed, err := m.SetFirewall(ctx, st)
|
||||
switch {
|
||||
case err != nil:
|
||||
p.log.Warn("could not tell the mirror what the settings say of its firewall", "err", err)
|
||||
case changed:
|
||||
p.log.Info("the mirror's firewall follows the cluster's settings", "enabled", st.Enabled)
|
||||
}
|
||||
}
|
||||
if p.st != nil {
|
||||
b, _ := json.Marshal(st)
|
||||
changed, err := p.st.PutClusterState(ctx, firewallStateKey, string(b))
|
||||
switch {
|
||||
case err != nil:
|
||||
p.log.Warn("could not record the firewall's state for the store's members", "err", err)
|
||||
case changed:
|
||||
p.log.Info("the store's members' firewall follows the cluster's settings", "enabled", st.Enabled)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
func TestTheLeaderTellsTheMirrorAndTheStoreWhatTheFirewallSettingsSay(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "s.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer st.Close()
|
||||
if err := st.PutControlPlane(ctx, store.ControlPlane{URL: "https://10.0.0.150:7443", Name: "ctl02"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root := t.TempDir()
|
||||
srv := httptest.NewServer(mirror.NewAPI(mirror.NewManager(root, slog.New(slog.DiscardHandler)), "http://m:8081").Handler())
|
||||
defer srv.Close()
|
||||
mc, _ := mirror.NewClient(srv.URL, tlsconf.Config{}, 0)
|
||||
|
||||
cfg := config.Server{Firewall: config.FirewallSettings{Enabled: false}, FirewallNetworks: []string{"192.168.5.0/24"}}
|
||||
leader := false
|
||||
p := newFirewallPublisher(func() config.Server { return cfg }, st,
|
||||
func() []string { return []string{"10.0.0.147", "", "10.0.0.147", "node-by-name"} },
|
||||
func() bool { return leader }, slog.New(slog.DiscardHandler))
|
||||
p.setMirror(mc)
|
||||
|
||||
p.once(ctx)
|
||||
if _, err := st.ClusterState(ctx, firewallStateKey); err == nil {
|
||||
t.Error("a standby told the store")
|
||||
}
|
||||
leader = true
|
||||
p.once(ctx)
|
||||
var got mirror.FirewallState
|
||||
v, err := st.ClusterState(ctx, firewallStateKey)
|
||||
if err != nil || json.Unmarshal([]byte(v), &got) != nil {
|
||||
t.Fatalf("the store holds %q %v", v, err)
|
||||
}
|
||||
if got.Enabled || got.Networks[0] != "192.168.5.0/24" || len(got.Toward) != 2 ||
|
||||
got.Toward[0] != "10.0.0.147" || got.Toward[1] != "10.0.0.150" {
|
||||
t.Errorf("state %+v", got)
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(root, mirror.FirewallFile))
|
||||
if err != nil || string(b) != v {
|
||||
t.Errorf("the mirror holds %s, the store %s", b, v)
|
||||
}
|
||||
}
|
||||
+226
-66
@@ -4,20 +4,18 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
)
|
||||
|
||||
// The judge the guardrails are installed with: Meta's Llama Guard 3 1B,
|
||||
// from the repository server init fetches it from.
|
||||
const (
|
||||
guardModelRepo = "QuantFactory/Llama-Guard-3-1B-GGUF"
|
||||
guardModelFile = "Llama-Guard-3-1B.Q4_K_M.gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
)
|
||||
|
||||
func newGuardJudgeCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
@@ -27,9 +25,15 @@ func newGuardJudgeCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
Long: "A policy with a judge has a model read what patterns cannot. The judge must\n" +
|
||||
"be in the catalog under the name the policy gives, as a judge (role:\n" +
|
||||
"guard), with a replica ready; otherwise what the policy judges is refused\n" +
|
||||
"(fail: closed, guard_unavailable) or served unjudged (fail: open).",
|
||||
"(fail: closed, guard_unavailable) or served unjudged (fail: open).\n\n" +
|
||||
"A policy may name several, asked in turn (judge.models): a judge on a GPU\n" +
|
||||
"first and one on CPUs after it is the GPU's speed while it is there, and\n" +
|
||||
"the CPUs' when it is not.\n\n" +
|
||||
"Judges llmgw knows where to fetch, which a policy may name and the leader\n" +
|
||||
"then fetches and serves by itself: " + knownJudgeNames() + ".",
|
||||
}
|
||||
c.AddCommand(newGuardJudgeStatusCmd(client, asJSON), newGuardJudgeEnableCmd(client), newGuardJudgeAskCmd(client, asJSON))
|
||||
c.AddCommand(newGuardJudgeStatusCmd(client, asJSON), newGuardJudgeEnableCmd(client),
|
||||
newGuardJudgeUseCmd(client), newGuardJudgeAskCmd(client, asJSON))
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -61,23 +65,25 @@ func newGuardJudgeStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Comma
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d/%d\t%s\n", j.Model, strings.Join(j.Policies, ","),
|
||||
yesNo(j.InCatalog), yesNo(j.Judge), j.Ready, j.Wanted, state)
|
||||
}
|
||||
for _, j := range js {
|
||||
if len(j.Then) > 0 {
|
||||
fmt.Fprintf(w, "\n%s gives no verdict: %s is asked instead", j.Model, strings.Join(j.Then, ", else "))
|
||||
}
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
strays, _ := strayJudges(cmd, client, js)
|
||||
for _, s := range strays {
|
||||
fmt.Fprintf(out, "\n%s looks like a judge's weights but is served as a model clients use, and no "+
|
||||
"policy asks it: llmgw ctl model remove %s\n", s, s)
|
||||
}
|
||||
sayStrays(out, strays)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// strayJudges are models whose weights are a judge's but which are in the
|
||||
// catalog as models clients use, under a name no policy asks: what serving
|
||||
// Llama Guard from the mirror page did before it knew better.
|
||||
func strayJudges(cmd *cobra.Command, client func() *ctlClient, js []adminapi.JudgeView) ([]string, error) {
|
||||
// strayJudges are models whose weights are a judge's, under a name no policy
|
||||
// asks: a judge the policies moved on from, or one served from the mirror
|
||||
// page as a model clients use, as before that page knew better.
|
||||
func strayJudges(cmd *cobra.Command, client func() *ctlClient, js []adminapi.JudgeView) ([]adminapi.ModelView, error) {
|
||||
var models []adminapi.ModelView
|
||||
if err := client().get(cmd.Context(), "/api/v1/models", &models); err != nil {
|
||||
return nil, err
|
||||
@@ -86,33 +92,52 @@ func strayJudges(cmd *cobra.Command, client func() *ctlClient, js []adminapi.Jud
|
||||
for _, j := range js {
|
||||
asked[j.Model] = true
|
||||
}
|
||||
var out []string
|
||||
var out []adminapi.ModelView
|
||||
for _, m := range models {
|
||||
if !asked[m.Name] && strings.Contains(strings.ToLower(m.Source), "llama-guard") {
|
||||
out = append(out, m.Name)
|
||||
if _, judge := guard.FormatOfWeights(m.Source); !asked[m.Name] && judge {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sayStrays says what each stray judge is and how to be rid of it.
|
||||
func sayStrays(out io.Writer, strays []adminapi.ModelView) {
|
||||
for _, m := range strays {
|
||||
if m.Role == catalog.RoleGuard {
|
||||
fmt.Fprintf(out, "\n%s is a judge no policy asks, still running: llmgw ctl model remove %s\n", m.Name, m.Name)
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(out, "\n%s looks like a judge's weights but is served as a model clients use, and no "+
|
||||
"policy asks it: llmgw ctl model remove %s\n", m.Name, m.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func newGuardJudgeEnableCmd(client func() *ctlClient) *cobra.Command {
|
||||
var (
|
||||
model, repo, file string
|
||||
replicas, cores int
|
||||
replace, detach bool
|
||||
model, repo, file, format, gpuClass string
|
||||
replicas, cores int
|
||||
gpu, replace, detach bool
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "enable",
|
||||
Short: "Serve the judge the policies ask, as a judge: Llama Guard 3 1B on CPUs",
|
||||
Long: "Fetches Llama Guard 3 1B into the mirror (once), adds it to the catalog under\n" +
|
||||
"the name the policies give its judge, as a judge (role: guard,\n" +
|
||||
"format llamaguard3), on CPUs so it takes no card, and waits for a replica\n" +
|
||||
"to answer. With no --model, the judge the policies ask that is missing.\n\n" +
|
||||
"A model of that name already in the catalog as a model clients use is\n" +
|
||||
"refused; --replace removes it first. An offline cluster gets its judge\n" +
|
||||
"from its bundle with server init.",
|
||||
Short: "Serve a judge: Qwen3Guard-Gen 0.6B on CPUs, or with --gpu the 4B on a GPU",
|
||||
Long: "Fetches a judge's weights into the mirror (once; what is there already, its\n" +
|
||||
"checksums matching, is not fetched again), adds it to the catalog as a judge\n" +
|
||||
"(role: guard) and waits for a replica to answer. Which judge: --model, or\n" +
|
||||
"with --gpu " + defaultGPUJudge + ", or else the first the policies ask that is\n" +
|
||||
"missing, or else " + defaultJudge + ".\n\n" +
|
||||
"A judge on CPUs takes no card. One on a GPU takes one card, as a standard\n" +
|
||||
"model: from nothing more important, so it waits while the cards are busy;\n" +
|
||||
"put a judge on CPUs after it in the policies (llmgw ctl guard judge use) so\n" +
|
||||
"there is always one to ask.\n\n" +
|
||||
"Known judges: " + knownJudgeNames() + ". Another is served with --repo,\n" +
|
||||
"--file and --format. A model of that name already in the catalog as a\n" +
|
||||
"model clients use is refused; --replace removes it first. An offline\n" +
|
||||
"cluster gets its judge from its bundle with server init.",
|
||||
Example: " llmgw ctl guard judge enable\n" +
|
||||
" llmgw ctl guard judge enable --replace",
|
||||
" llmgw ctl guard judge enable --gpu\n" +
|
||||
" llmgw ctl guard judge use " + defaultGPUJudge + " " + defaultJudge,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
ctx := cmd.Context()
|
||||
@@ -121,45 +146,72 @@ func newGuardJudgeEnableCmd(client func() *ctlClient) *cobra.Command {
|
||||
if err := client().get(ctx, "/api/v1/guard/judges", &js); err != nil {
|
||||
return err
|
||||
}
|
||||
var target *adminapi.JudgeView
|
||||
for i, j := range js {
|
||||
if (model != "" && j.Model == model) || (model == "" && (!j.InCatalog || !j.Judge)) {
|
||||
target = &js[i]
|
||||
break
|
||||
name := model
|
||||
if name == "" && gpu {
|
||||
name = defaultGPUJudge
|
||||
}
|
||||
if name == "" {
|
||||
for _, j := range js {
|
||||
if !j.InCatalog || !j.Judge {
|
||||
name = j.Model
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if name == "" {
|
||||
name = defaultJudge
|
||||
}
|
||||
|
||||
k, known := knownJudgeNamed(name)
|
||||
switch {
|
||||
case target == nil && model != "":
|
||||
target = &adminapi.JudgeView{Model: model}
|
||||
fmt.Fprintf(out, "no policy asks %s yet; it is served, and a policy's judge.model names it\n", model)
|
||||
case target == nil && len(js) == 0:
|
||||
return errors.New("no policy asks a judge: give one a judge (llmgw ctl config edit, guard.policies.<name>.judge.model)")
|
||||
case target == nil:
|
||||
fmt.Fprintln(out, "every judge the policies ask is in the catalog, as a judge")
|
||||
return nil
|
||||
case repo != "":
|
||||
f := format
|
||||
if f == "" {
|
||||
f, _ = guard.FormatOfWeights(repo + "/" + file)
|
||||
}
|
||||
if f == "" || !guard.KnownFormat(f) {
|
||||
return fmt.Errorf("say how its answers are read: --format %s", strings.Join(guard.Formats, "|"))
|
||||
}
|
||||
k = knownJudge{Name: name, Repo: repo, File: file, Format: f, GPU: gpu}
|
||||
case !known:
|
||||
return fmt.Errorf("%s is not a judge llmgw knows where to fetch (%s): give --repo, --file and --format", name, knownJudgeNames())
|
||||
}
|
||||
if target.InCatalog && target.Judge {
|
||||
fmt.Fprintf(out, "%s is in the catalog as a judge already; llmgw ctl guard judge status says how it is\n", target.Model)
|
||||
return nil
|
||||
if gpu {
|
||||
k.GPU = true
|
||||
}
|
||||
if target.InCatalog && !target.Judge {
|
||||
|
||||
var models []adminapi.ModelView
|
||||
if err := client().get(ctx, "/api/v1/models", &models); err != nil {
|
||||
return err
|
||||
}
|
||||
if i := slices.IndexFunc(models, func(m adminapi.ModelView) bool { return m.Name == name }); i >= 0 {
|
||||
if models[i].Role == catalog.RoleGuard {
|
||||
fmt.Fprintf(out, "%s is in the catalog as a judge already; llmgw ctl guard judge status says how it is\n", name)
|
||||
return nil
|
||||
}
|
||||
if !replace {
|
||||
return fmt.Errorf("%s is in the catalog as a model clients use, not as a judge: "+
|
||||
"--replace removes it and serves it again as a judge", target.Model)
|
||||
"--replace removes it and serves it again as a judge", name)
|
||||
}
|
||||
var res map[string]string
|
||||
if err := client().delete(ctx, "/api/v1/models/"+target.Model, &res); err != nil {
|
||||
return fmt.Errorf("removing %s: %w", target.Model, err)
|
||||
if err := client().delete(ctx, "/api/v1/models/"+name, &res); err != nil {
|
||||
return fmt.Errorf("removing %s: %w", name, err)
|
||||
}
|
||||
fmt.Fprintf(out, "%s removed\n", target.Model)
|
||||
fmt.Fprintf(out, "%s removed\n", name)
|
||||
}
|
||||
req := modelserve.Request{
|
||||
Repo: repo, File: file, Name: target.Model, GuardFormat: "llamaguard3",
|
||||
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
|
||||
CPU: true, Replicas: replicas, CPUCores: cores,
|
||||
CtxPool: 8192, ContextLength: 4096,
|
||||
|
||||
opts := judgeServeOptions{Replicas: replicas, Cores: cores, GPUClass: gpuClass}
|
||||
if k.GPU && opts.GPUClass == "" {
|
||||
var s adminapi.Summary
|
||||
if err := client().get(ctx, "/api/v1/summary", &s); err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.GPUClass = roomiestGPUClass(s.GPUs); opts.GPUClass == "" {
|
||||
return fmt.Errorf("this cluster has no GPU for %s: serve %s on CPUs instead", name, defaultJudge)
|
||||
}
|
||||
}
|
||||
body, _ := json.Marshal(req)
|
||||
fmt.Fprintf(out, "serving %s\n", k)
|
||||
body, _ := json.Marshal(k.serveRequest(opts))
|
||||
var job modelserve.Job
|
||||
if err := client().patient(30*time.Second).post(ctx, "/api/v1/models/serve", body, &job); err != nil {
|
||||
return err
|
||||
@@ -170,24 +222,132 @@ func newGuardJudgeEnableCmd(client func() *ctlClient) *cobra.Command {
|
||||
if err := followJob(cmd, client, job, "is the judge"); err != nil {
|
||||
return err
|
||||
}
|
||||
if !slices.ContainsFunc(js, func(j adminapi.JudgeView) bool { return j.Model == name }) {
|
||||
fmt.Fprintf(out, "\nno policy asks %s yet: llmgw ctl guard judge use %s", name, name)
|
||||
if k.GPU {
|
||||
fmt.Fprintf(out, " %s", defaultJudge)
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
if strays, _ := strayJudges(cmd, client, js); len(strays) > 0 {
|
||||
fmt.Fprintf(out, "\nno policy asks %s, served from a judge's weights as a model clients use: "+
|
||||
"llmgw ctl model remove %s\n", strings.Join(strays, ", "), strings.Join(strays, " "))
|
||||
sayStrays(out, slices.DeleteFunc(strays, func(m adminapi.ModelView) bool { return m.Name == name }))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
f := c.Flags()
|
||||
f.StringVar(&model, "model", "", "the judge to serve (default: the one the policies ask that is missing)")
|
||||
f.StringVar(&repo, "repo", guardModelRepo, "the registry repository its weights come from")
|
||||
f.StringVar(&file, "file", guardModelFile, "the file in it")
|
||||
f.IntVar(&replicas, "replicas", 2, "replicas: every judged request waits on one")
|
||||
f.IntVar(&cores, "cores", 4, "CPU cores a replica computes on")
|
||||
f.StringVar(&model, "model", "", "the judge to serve: "+knownJudgeNames()+", or a name for --repo's")
|
||||
f.BoolVar(&gpu, "gpu", false, "serve it on a GPU ("+defaultGPUJudge+" when no --model)")
|
||||
f.StringVar(&gpuClass, "gpu-class", "", "the GPUs it runs on (default: the class with the most free cards)")
|
||||
f.StringVar(&repo, "repo", "", "the registry repository of a judge llmgw does not know")
|
||||
f.StringVar(&file, "file", "", "the GGUF file in --repo")
|
||||
f.StringVar(&format, "format", "", "how --repo's judge answers: "+strings.Join(guard.Formats, ", "))
|
||||
f.IntVar(&replicas, "replicas", 0, "replicas (default: 2 on CPUs, as every judged request waits on one; 1 on a GPU)")
|
||||
f.IntVar(&cores, "cores", 4, "CPU cores a replica on CPUs computes on")
|
||||
f.BoolVar(&replace, "replace", false, "remove a model of that name that is not a judge first")
|
||||
f.BoolVar(&detach, "detach", false, "start it and return; llmgw ctl jobs follows it")
|
||||
return c
|
||||
}
|
||||
|
||||
// roomiestGPUClass is the class with the most free cards, else the most
|
||||
// cards; empty when the cluster has none.
|
||||
func roomiestGPUClass(classes []adminapi.ClassUse) string {
|
||||
best := -1
|
||||
for i, c := range classes {
|
||||
if c.Total == 0 {
|
||||
continue
|
||||
}
|
||||
if best < 0 || c.Free() > classes[best].Free() || (c.Free() == classes[best].Free() && c.Total > classes[best].Total) {
|
||||
best = i
|
||||
}
|
||||
}
|
||||
if best < 0 {
|
||||
return ""
|
||||
}
|
||||
return classes[best].Class
|
||||
}
|
||||
|
||||
func newGuardJudgeUseCmd(client func() *ctlClient) *cobra.Command {
|
||||
var policies []string
|
||||
var note string
|
||||
c := &cobra.Command{
|
||||
Use: "use JUDGE...",
|
||||
Short: "Set the judges the policies ask, in the order they are asked",
|
||||
Long: "Sets judge.models of every policy that has a judge, or of those given with\n" +
|
||||
"--policy: the first judge that answers gives the verdict, and the next is\n" +
|
||||
"asked when one has no replica serving, fails, or does not answer within\n" +
|
||||
"the policy's latency_budget. A judge on a GPU first and one on CPUs after\n" +
|
||||
"it is the GPU's speed while it is there, and the CPUs' when it is not.\n\n" +
|
||||
"A judge llmgw knows (" + knownJudgeNames() + ")\n" +
|
||||
"that is not in the catalog is fetched and served by the leader; another\n" +
|
||||
"is served first with llmgw ctl guard judge enable --repo. In force on\n" +
|
||||
"every control plane within seconds.",
|
||||
Example: " llmgw ctl guard judge use " + defaultJudge + "\n" +
|
||||
" llmgw ctl guard judge use " + defaultGPUJudge + " " + defaultJudge + "\n" +
|
||||
" llmgw ctl guard judge use " + defaultGPUJudge + " " + defaultJudge + " --policy strict",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
for i, a := range args {
|
||||
if slices.Contains(args[:i], a) {
|
||||
return fmt.Errorf("%s is named twice", a)
|
||||
}
|
||||
}
|
||||
if note == "" {
|
||||
note = "judges: " + strings.Join(args, ", else ")
|
||||
}
|
||||
list := "[" + strings.Join(args, ", ") + "]"
|
||||
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
|
||||
names := policies
|
||||
if len(names) == 0 {
|
||||
for _, p := range doc.MapKeys([]string{"guard", "policies"}) {
|
||||
if doc.Node([]string{"guard", "policies", p, "judge"}) != nil {
|
||||
names = append(names, p)
|
||||
}
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return errors.New("no policy has a judge: name one with --policy, and it is given one " +
|
||||
"that blocks what the judge finds unsafe in requests (judge.input: block)")
|
||||
}
|
||||
}
|
||||
for _, p := range names {
|
||||
base := []string{"guard", "policies", p}
|
||||
if doc.Node(base) == nil {
|
||||
return fmt.Errorf("there is no policy %s (llmgw ctl guard list)", p)
|
||||
}
|
||||
judge := append(slices.Clone(base), "judge")
|
||||
if doc.Node(judge) == nil {
|
||||
if err := doc.SetYAML(append(slices.Clone(judge), "input"), "block"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
doc.Unset(append(slices.Clone(judge), "model"))
|
||||
if err := doc.SetYAML(append(slices.Clone(judge), "models"), list); err != nil {
|
||||
return err
|
||||
}
|
||||
cats := append(slices.Clone(judge), "categories")
|
||||
var have []string
|
||||
if n := doc.Node(cats); n != nil {
|
||||
for _, c := range n.Content {
|
||||
have = append(have, c.Value)
|
||||
}
|
||||
}
|
||||
if next := categoriesFor(args, have); next != nil {
|
||||
if err := doc.SetYAML(cats, "["+strings.Join(next, ", ")+"]"); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s: categories [%s] are not what %s name; it counts [%s] instead\n",
|
||||
p, strings.Join(have, ", "), strings.Join(args, " or "), strings.Join(next, ", "))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
},
|
||||
}
|
||||
c.Flags().StringSliceVar(&policies, "policy", nil, "the policies to change (default: every one with a judge)")
|
||||
c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version")
|
||||
return c
|
||||
}
|
||||
|
||||
func newGuardJudgeAskCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
var model, answer string
|
||||
c := &cobra.Command{
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
// judgeKeeper fetches and serves, on the leader, each judge llmgw knows that
|
||||
// a policy names and the catalog does not have: naming qwen3guard-4b in a
|
||||
// policy is all it takes to have one, and a judge removed by mistake comes
|
||||
// back. A judge it does not know is left to llmgw ctl guard judge enable
|
||||
// --repo, said in guard judge status.
|
||||
type judgeKeeper struct {
|
||||
library func() guard.Config
|
||||
models func() []catalog.Model
|
||||
serve interface {
|
||||
Start(modelserve.Request) (modelserve.Job, error)
|
||||
Jobs() []modelserve.Job
|
||||
}
|
||||
leader func() bool
|
||||
gpus func() []adminapi.ClassUse
|
||||
log *slog.Logger
|
||||
now func() time.Time
|
||||
|
||||
// tried is when each judge was last asked for, so one that cannot be
|
||||
// fetched is not asked for again on every look.
|
||||
tried map[string]time.Time
|
||||
}
|
||||
|
||||
// judgeRetry is how long a judge whose serving failed waits before it is
|
||||
// asked for again.
|
||||
const judgeRetry = 10 * time.Minute
|
||||
|
||||
func (k *judgeKeeper) run(ctx context.Context, every time.Duration) {
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
k.check()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (k *judgeKeeper) check() {
|
||||
if k.leader != nil && !k.leader() {
|
||||
return
|
||||
}
|
||||
lib := k.library()
|
||||
if !lib.On() {
|
||||
return
|
||||
}
|
||||
if k.tried == nil {
|
||||
k.tried = map[string]time.Time{}
|
||||
}
|
||||
now := time.Now
|
||||
if k.now != nil {
|
||||
now = k.now
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, m := range k.models() {
|
||||
have[m.Name] = true
|
||||
}
|
||||
busy := map[string]bool{}
|
||||
for _, j := range k.serve.Jobs() {
|
||||
if j.State == modelserve.Running {
|
||||
busy[j.Model] = true
|
||||
busy[j.Request.Name] = true
|
||||
}
|
||||
}
|
||||
var wanted []string
|
||||
for _, p := range lib.Policies {
|
||||
for _, j := range p.Judge.Judges() {
|
||||
if !slices.Contains(wanted, j) {
|
||||
wanted = append(wanted, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.Sort(wanted)
|
||||
for _, name := range wanted {
|
||||
kj, known := knownJudgeNamed(name)
|
||||
if have[name] || !known || busy[name] {
|
||||
continue
|
||||
}
|
||||
if at, ok := k.tried[name]; ok && now().Sub(at) < judgeRetry {
|
||||
continue
|
||||
}
|
||||
k.tried[name] = now()
|
||||
var opts judgeServeOptions
|
||||
if kj.GPU {
|
||||
if opts.GPUClass = roomiestGPUClass(k.gpus()); opts.GPUClass == "" {
|
||||
k.log.Warn("a policy names a judge that runs on a GPU, and this cluster has none",
|
||||
"judge", name, "instead", defaultJudge)
|
||||
continue
|
||||
}
|
||||
}
|
||||
req := kj.serveRequest(opts)
|
||||
req.By = "llmgw (a guard policy names it)"
|
||||
job, err := k.serve.Start(req)
|
||||
if err != nil {
|
||||
k.log.Error("could not start serving a judge a policy names", "judge", name, "err", err)
|
||||
continue
|
||||
}
|
||||
k.log.Info("serving a judge a policy names, which the catalog did not have",
|
||||
"judge", name, "repo", kj.Repo, "file", kj.File, "job", job.ID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
type fakeServe struct {
|
||||
started []modelserve.Request
|
||||
running []modelserve.Job
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeServe) Start(r modelserve.Request) (modelserve.Job, error) {
|
||||
if f.err != nil {
|
||||
return modelserve.Job{}, f.err
|
||||
}
|
||||
f.started = append(f.started, r)
|
||||
return modelserve.Job{ID: "j", Model: r.Name, Request: r, State: modelserve.Running}, nil
|
||||
}
|
||||
func (f *fakeServe) Jobs() []modelserve.Job { return f.running }
|
||||
|
||||
func TestAJudgeAPolicyNamesIsFetchedWhenTheCatalogLacksIt(t *testing.T) {
|
||||
lib := guard.Config{Policies: map[string]guard.Policy{
|
||||
"baseline": {Judge: &guard.JudgePolicy{Models: []string{"qwen3guard-4b", "qwen3guard-0.6b"}, Input: "block"}},
|
||||
"custom": {Judge: &guard.JudgePolicy{Model: "our-own-judge", Input: "block"}},
|
||||
}}
|
||||
inCatalog := []catalog.Model{{Name: "qwen3guard-0.6b"}}
|
||||
now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
fs := &fakeServe{}
|
||||
leader := true
|
||||
k := &judgeKeeper{
|
||||
library: func() guard.Config { return lib },
|
||||
models: func() []catalog.Model { return inCatalog },
|
||||
serve: fs,
|
||||
leader: func() bool { return leader },
|
||||
gpus: func() []adminapi.ClassUse { return []adminapi.ClassUse{{Class: "6000-24g", Total: 1, Used: 1}} },
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
now: func() time.Time { return now },
|
||||
}
|
||||
|
||||
k.check()
|
||||
// The GPU judge alone: the CPU one is there, and one llmgw does not know
|
||||
// is not guessed at.
|
||||
if len(fs.started) != 1 || fs.started[0].Name != "qwen3guard-4b" || fs.started[0].GPUClass != "6000-24g" ||
|
||||
fs.started[0].Class != catalog.ClassStandard || fs.started[0].GuardFormat != "qwen3guard" || fs.started[0].CPU {
|
||||
t.Fatalf("started %+v", fs.started)
|
||||
}
|
||||
|
||||
// Being fetched: not asked for again.
|
||||
fs.running = []modelserve.Job{{Model: "qwen3guard-4b", State: modelserve.Running}}
|
||||
k.check()
|
||||
if len(fs.started) != 1 {
|
||||
t.Errorf("asked for again while it is being fetched: %d", len(fs.started))
|
||||
}
|
||||
// Failed: asked for again only after a while.
|
||||
fs.running = nil
|
||||
now = now.Add(time.Minute)
|
||||
k.check()
|
||||
if len(fs.started) != 1 {
|
||||
t.Error("asked for again at once after it failed")
|
||||
}
|
||||
now = now.Add(judgeRetry)
|
||||
k.check()
|
||||
if len(fs.started) != 2 {
|
||||
t.Error("never asked for again")
|
||||
}
|
||||
|
||||
// Only the leader fetches, and nothing while the guardrails are off.
|
||||
leader = false
|
||||
now = now.Add(judgeRetry)
|
||||
k.check()
|
||||
leader = true
|
||||
off := false
|
||||
lib.Enabled = &off
|
||||
k.check()
|
||||
if len(fs.started) != 2 {
|
||||
t.Errorf("a standby or the guardrails off fetched: %d", len(fs.started))
|
||||
}
|
||||
|
||||
// A CPU judge on a cluster without GPUs is served on CPUs, two of it.
|
||||
lib = guard.Config{Policies: map[string]guard.Policy{"b": {Judge: &guard.JudgePolicy{Model: "qwen3guard-0.6b", Input: "block"}}}}
|
||||
inCatalog = nil
|
||||
k.gpus = func() []adminapi.ClassUse { return nil }
|
||||
k.check()
|
||||
if last := fs.started[len(fs.started)-1]; last.Name != "qwen3guard-0.6b" || !last.CPU || last.Replicas != 2 {
|
||||
t.Errorf("CPU judge: %+v", last)
|
||||
}
|
||||
fs.err = errors.New("the mirror is down")
|
||||
lib = guard.Config{Policies: map[string]guard.Policy{"b": {Judge: &guard.JudgePolicy{Model: "qwen3guard-8b", Input: "block"}}}}
|
||||
k.check() // no GPU, so not started, and no panic on the error path either
|
||||
}
|
||||
|
||||
// A policy moved from Llama Guard's codes to Qwen3Guard would count nothing:
|
||||
// its categories become the judge's own.
|
||||
func TestCategoriesFollowTheJudges(t *testing.T) {
|
||||
lg := []string{"S1", "S2", "S10"}
|
||||
if got := categoriesFor([]string{"qwen3guard-0.6b"}, lg); !slices.Equal(got, baselineCategories["qwen3guard"]) {
|
||||
t.Errorf("Llama Guard's codes for Qwen3Guard: %v", got)
|
||||
}
|
||||
if got := categoriesFor([]string{"qwen3guard-4b", "qwen3guard-0.6b"}, []string{"Violent", "Jailbreak"}); got != nil {
|
||||
t.Errorf("its own names were replaced: %v", got)
|
||||
}
|
||||
if got := categoriesFor([]string{"llama-guard-3-1b"}, lg); got != nil {
|
||||
t.Errorf("Llama Guard's own codes were replaced: %v", got)
|
||||
}
|
||||
// Both kinds asked in turn: each must have something to count.
|
||||
got := categoriesFor([]string{"qwen3guard-4b", "llama-guard-3-1b"}, []string{"Violent"})
|
||||
if !slices.Contains(got, "Violent") || !slices.Contains(got, "S1") {
|
||||
t.Errorf("mixed: %v", got)
|
||||
}
|
||||
if got := categoriesFor([]string{"our-own-judge"}, lg); got != nil {
|
||||
t.Errorf("a judge llmgw does not know: %v", got)
|
||||
}
|
||||
if got := categoriesFor([]string{"qwen3guard-0.6b"}, nil); got != nil {
|
||||
t.Errorf("no categories (every one counts) became %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
// knownJudge is a judge model llmgw knows where to fetch and how to serve:
|
||||
// what a policy may name without anyone saying where its weights are.
|
||||
type knownJudge struct {
|
||||
Name, Repo, File, Format string
|
||||
// GPU is a judge too slow on CPUs to wait for: served on one card.
|
||||
GPU bool
|
||||
About string
|
||||
}
|
||||
|
||||
// knownJudges, the default first. Measured on the reference cluster against
|
||||
// ordinary coding requests ("continue", "kill the process on port 8080") and
|
||||
// harmful ones: Llama Guard 3 1B scored both alike, about one in two
|
||||
// ordinary ones refused; Qwen3Guard-Gen 0.6B kept them apart, 0.15 against
|
||||
// 1.00 at the worst, at about two seconds a check on two CPU threads.
|
||||
var knownJudges = []knownJudge{
|
||||
{Name: "qwen3guard-0.6b", Repo: "mradermacher/Qwen3Guard-Gen-0.6B-GGUF", File: "Qwen3Guard-Gen-0.6B.Q8_0.gguf",
|
||||
Format: "qwen3guard", About: "Qwen3Guard-Gen 0.6B on CPUs, 0.8 GB: the default"},
|
||||
{Name: "qwen3guard-4b", Repo: "mradermacher/Qwen3Guard-Gen-4B-GGUF", File: "Qwen3Guard-Gen-4B.Q4_K_M.gguf",
|
||||
Format: "qwen3guard", GPU: true, About: "Qwen3Guard-Gen 4B on a GPU, 2.5 GB"},
|
||||
{Name: "qwen3guard-8b", Repo: "mradermacher/Qwen3Guard-Gen-8B-GGUF", File: "Qwen3Guard-Gen-8B.Q4_K_M.gguf",
|
||||
Format: "qwen3guard", GPU: true, About: "Qwen3Guard-Gen 8B on a GPU, 4.7 GB"},
|
||||
{Name: "llama-guard-3-1b", Repo: "QuantFactory/Llama-Guard-3-1B-GGUF", File: "Llama-Guard-3-1B.Q4_K_M.gguf",
|
||||
Format: "llamaguard3", About: "Llama Guard 3 1B on CPUs, 0.9 GB: refuses ordinary coding requests"},
|
||||
{Name: "llama-guard-3-8b", Repo: "QuantFactory/Llama-Guard-3-8B-GGUF", File: "Llama-Guard-3-8B.Q4_K_M.gguf",
|
||||
Format: "llamaguard3", GPU: true, About: "Llama Guard 3 8B on a GPU, 4.6 GB"},
|
||||
}
|
||||
|
||||
const (
|
||||
defaultJudge = "qwen3guard-0.6b"
|
||||
defaultGPUJudge = "qwen3guard-4b"
|
||||
)
|
||||
|
||||
// baselineCategories are what the baseline policy counts of each format's
|
||||
// verdicts: not PII, which is the patterns', nor what refuses ordinary work
|
||||
// — copyright, unethical acts, politically sensitive topics; Llama Guard's
|
||||
// defamation, specialized advice, intellectual property and elections.
|
||||
var baselineCategories = map[string][]string{
|
||||
"qwen3guard": {"Violent", "Non-violent Illegal Acts", "Sexual Content or Sexual Acts", "Suicide & Self-Harm", "Jailbreak"},
|
||||
"llamaguard3": {"S1", "S2", "S3", "S4", "S7", "S9", "S10", "S11", "S12"},
|
||||
}
|
||||
|
||||
// reported is whether a judge of that format ever names category c: its own
|
||||
// names, or for Llama Guard its codes.
|
||||
func reported(format, c string) bool {
|
||||
if slices.ContainsFunc(guard.DefaultCategories(format), func(n string) bool { return strings.EqualFold(n, c) }) {
|
||||
return true
|
||||
}
|
||||
var n int
|
||||
_, err := fmt.Sscanf(c, "S%d", &n)
|
||||
return format == "llamaguard3" && err == nil && n >= 1 && n <= 14
|
||||
}
|
||||
|
||||
// categoriesFor is what a policy asking these judges should count, when the
|
||||
// categories it has are ones none of them names: a policy moved from Llama
|
||||
// Guard's S1 to Qwen3Guard, which says Violent, would otherwise count
|
||||
// nothing. Nil keeps what it has; so does a judge llmgw does not know.
|
||||
func categoriesFor(judges, have []string) []string {
|
||||
var formats []string
|
||||
for _, j := range judges {
|
||||
k, ok := knownJudgeNamed(j)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if !slices.Contains(formats, k.Format) {
|
||||
formats = append(formats, k.Format)
|
||||
}
|
||||
}
|
||||
if len(have) == 0 {
|
||||
return nil
|
||||
}
|
||||
stray := false
|
||||
for _, f := range formats {
|
||||
for _, c := range have {
|
||||
if !reported(f, c) && !slices.ContainsFunc(formats, func(o string) bool { return reported(o, c) }) {
|
||||
stray = true
|
||||
}
|
||||
}
|
||||
// Every judge must have something it counts.
|
||||
if !slices.ContainsFunc(have, func(c string) bool { return reported(f, c) }) {
|
||||
stray = true
|
||||
}
|
||||
}
|
||||
if !stray {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, f := range formats {
|
||||
out = append(out, baselineCategories[f]...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func knownJudgeNamed(name string) (knownJudge, bool) {
|
||||
i := slices.IndexFunc(knownJudges, func(k knownJudge) bool { return k.Name == name })
|
||||
if i < 0 {
|
||||
return knownJudge{}, false
|
||||
}
|
||||
return knownJudges[i], true
|
||||
}
|
||||
|
||||
func knownJudgeNames() string {
|
||||
var n []string
|
||||
for _, k := range knownJudges {
|
||||
n = append(n, k.Name)
|
||||
}
|
||||
return strings.Join(n, ", ")
|
||||
}
|
||||
|
||||
// judgeServeOptions are how a judge is served beyond which weights.
|
||||
type judgeServeOptions struct {
|
||||
Replicas, Cores int
|
||||
// GPUClass is the cards a GPU judge runs on; empty lets the plan choose.
|
||||
GPUClass string
|
||||
}
|
||||
|
||||
// serveRequest is the serve job for a judge. A judge on CPUs is critical,
|
||||
// as every judged request waits on it; one on a GPU is standard, so it takes
|
||||
// a card only from what may give one up, and the judge on CPUs after it in
|
||||
// the policies stands in while it has none.
|
||||
func (k knownJudge) serveRequest(o judgeServeOptions) modelserve.Request {
|
||||
req := modelserve.Request{
|
||||
Repo: k.Repo, File: k.File, Name: k.Name, GuardFormat: k.Format,
|
||||
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
|
||||
Replicas: o.Replicas, CtxPool: 8192, ContextLength: 4096,
|
||||
}
|
||||
if k.GPU {
|
||||
req.GPUClass, req.GPUsPerNode, req.Class = o.GPUClass, 1, catalog.ClassStandard
|
||||
if req.Replicas == 0 {
|
||||
req.Replicas = 1
|
||||
}
|
||||
return req
|
||||
}
|
||||
req.CPU, req.CPUCores = true, o.Cores
|
||||
if req.Replicas == 0 {
|
||||
req.Replicas = 2
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
func (k knownJudge) String() string {
|
||||
where := "CPUs"
|
||||
if k.GPU {
|
||||
where = "a GPU"
|
||||
}
|
||||
return fmt.Sprintf("%s (%s, %s/%s, on %s)", k.Name, k.Format, k.Repo, k.File, where)
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
func newKBCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
Use: "kb",
|
||||
Short: "Knowledge bases: collections of documents requests are answered from",
|
||||
Long: "A collection belongs to an organization, team, project or person, and is\n" +
|
||||
"embedded by one model of the catalog. People put documents into it from\n" +
|
||||
"their own machines with llmgw-kb (its own download, for Windows, macOS and\n" +
|
||||
"Linux), which sends text and vectors, never the files.\n\n" +
|
||||
"Who may read or add to a collection: its owner and everything below it,\n" +
|
||||
"and whoever it is granted to (grant), never outside its organization.\n" +
|
||||
"Targets are written org:Acme, team:Acme/hr, project:Acme/hr/docs (or with\n" +
|
||||
"IDs), key:KEY-ID or user:USER-ID. A collection is named by its ID, its name\n" +
|
||||
"when that is unique, or ORG/NAME.\n\n" +
|
||||
"Needs the PostgreSQL store with pgvector; `llmgw ctl kb list` says why not.",
|
||||
}
|
||||
c.AddCommand(&cobra.Command{
|
||||
Use: "list",
|
||||
Short: "Every collection: owner, model, documents and grants",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
var v adminapi.KBView
|
||||
if err := client().get(cmd.Context(), "/api/v1/kb", &v); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
if !v.Available {
|
||||
fmt.Fprintln(out, "Knowledge bases are off: "+v.Reason)
|
||||
return nil
|
||||
}
|
||||
if len(v.Collections) == 0 {
|
||||
fmt.Fprintln(out, "No collections yet (llmgw ctl kb create).")
|
||||
return nil
|
||||
}
|
||||
w := tab()
|
||||
fmt.Fprintln(w, "COLLECTION\tOWNER\tMODEL\tDOCUMENTS\tCHUNKS\tGRANTED TO")
|
||||
for _, c := range v.Collections {
|
||||
var grants []string
|
||||
for _, g := range c.Grants {
|
||||
grants = append(grants, or(g.Name, g.Target)+" ("+g.Perm+")")
|
||||
}
|
||||
fmt.Fprintf(w, "%s/%s\t%s\t%s (%d)\t%d\t%d\t%s\n", or(c.OrgName, c.Org), c.Name, or(c.OwnerName, c.Owner),
|
||||
c.EmbedModel, c.Dims, c.Documents, c.Chunks, strings.Join(grants, ", "))
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(v.Defaults) > 0 {
|
||||
fmt.Fprintln(out, "\nAnswered from by default, when a request names no collection:")
|
||||
w = tab()
|
||||
for _, d := range v.Defaults {
|
||||
fmt.Fprintf(w, " %s\t%s\n", or(d.Name, d.Target), strings.Join(d.Collections, ", "))
|
||||
}
|
||||
return w.Flush()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
var org string
|
||||
var dims int
|
||||
create := &cobra.Command{
|
||||
Use: "create OWNER NAME MODEL",
|
||||
Short: "Make a collection owned by an organization, team, project or person",
|
||||
Long: "MODEL is a catalog model with capabilities.embeddings. Its dimension is asked\n" +
|
||||
"of it, so it has to be able to serve now; --dims gives it instead.",
|
||||
Example: " llmgw ctl kb create team:Acme/hr policies bge-m3\n" +
|
||||
" llmgw ctl kb create user:USER-ID notes bge-m3 --org Acme",
|
||||
Args: cobra.ExactArgs(3),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
body, _ := json.Marshal(adminapi.KBCreateRequest{Owner: args[0], Name: args[1], Model: args[2], Org: org, Dims: dims})
|
||||
var v adminapi.KBCollectionView
|
||||
if err := client().do(cmd.Context(), "POST", "/api/v1/kb/collections", json.RawMessage(body), &v); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "collection %s (%s), embedded by %s in %d dimensions\n", v.Name, v.ID, v.EmbedModel, v.Dims)
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "people with access fill it with: llmgw-kb sync %s FOLDER\n", v.Name)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
create.Flags().StringVar(&org, "org", "", "the organization of a person's collection")
|
||||
create.Flags().IntVar(&dims, "dims", 0, "the model's embedding dimension, instead of asking it")
|
||||
c.AddCommand(create)
|
||||
|
||||
c.AddCommand(&cobra.Command{
|
||||
Use: "delete COLLECTION",
|
||||
Short: "Remove a collection, its documents and its grants",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if err := client().do(cmd.Context(), "DELETE", "/api/v1/kb/collections/"+url.PathEscape(args[0]), nil, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "collection %s removed\n", args[0])
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
for _, verb := range []struct {
|
||||
use, short, method string
|
||||
perm bool
|
||||
}{
|
||||
{"grant", "Let a scope and everything below it read a collection, or with --write add to it", "POST", true},
|
||||
{"revoke", "Take a grant away", "DELETE", false},
|
||||
} {
|
||||
var write bool
|
||||
cmd := &cobra.Command{
|
||||
Use: verb.use + " COLLECTION TARGET",
|
||||
Short: verb.short,
|
||||
Example: " llmgw ctl kb " + verb.use + " Acme/policies project:Acme/sales/crm",
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
req := adminapi.KBGrantRequest{Collection: args[0], Target: args[1], Perm: "read"}
|
||||
if write {
|
||||
req.Perm = "write"
|
||||
}
|
||||
body, _ := json.Marshal(req)
|
||||
if err := client().do(cmd.Context(), verb.method, "/api/v1/kb/grants", json.RawMessage(body), nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if verb.perm {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s may %s %s\n", args[1], req.Perm, args[0])
|
||||
} else {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s no longer granted %s\n", args[1], args[0])
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
if verb.perm {
|
||||
cmd.Flags().BoolVar(&write, "write", false, "add and remove documents too")
|
||||
}
|
||||
c.AddCommand(cmd)
|
||||
}
|
||||
|
||||
c.AddCommand(&cobra.Command{
|
||||
Use: "docs COLLECTION",
|
||||
Short: "A collection's documents",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var docs []adminapi.KBDocumentView
|
||||
if err := client().get(cmd.Context(), "/api/v1/kb/collections/"+url.PathEscape(args[0])+"/documents", &docs); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(docs)
|
||||
}
|
||||
w := tab()
|
||||
fmt.Fprintln(w, "DOCUMENT\tCHUNKS\tSIZE\tADDED\tBY")
|
||||
for _, d := range docs {
|
||||
fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\n", d.Name, d.Chunks, humanBytes(d.Bytes), d.AddedAt.Local().Format("2006-01-02 15:04"), d.AddedBy)
|
||||
}
|
||||
return w.Flush()
|
||||
},
|
||||
})
|
||||
c.AddCommand(&cobra.Command{
|
||||
Use: "rm COLLECTION DOCUMENT",
|
||||
Short: "Remove one document from a collection",
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
path := "/api/v1/kb/collections/" + url.PathEscape(args[0]) + "/documents?name=" + url.QueryEscape(args[1])
|
||||
if err := client().do(cmd.Context(), "DELETE", path, nil, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s removed from %s\n", args[1], args[0])
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
var clear bool
|
||||
def := &cobra.Command{
|
||||
Use: "default TARGET [COLLECTION...]",
|
||||
Short: "Answer a part of the tree's chat requests from collections, when they name none",
|
||||
Long: "Requests from TARGET and everything below it are then answered from these\n" +
|
||||
"collections without naming them (X-Llmgw-Knowledge). The narrowest default\n" +
|
||||
"wins: a key's over its project's, a project's over its team's. A request's own\n" +
|
||||
"header overrides it, and X-Llmgw-Knowledge: none turns it off. A collection a\n" +
|
||||
"caller may not read is left out for them, and a search that fails answers\n" +
|
||||
"without passages rather than refusing. In force within 30 seconds.",
|
||||
Example: " llmgw ctl kb default team:Acme/hr Acme/policies Acme/forms\n" +
|
||||
" llmgw ctl kb default team:Acme/hr --clear",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if clear != (len(args) == 1) {
|
||||
return fmt.Errorf("name collections, or --clear")
|
||||
}
|
||||
body, _ := json.Marshal(adminapi.KBDefaultRequest{Target: args[0], Collections: args[1:]})
|
||||
if err := client().do(cmd.Context(), "POST", "/api/v1/kb/defaults", json.RawMessage(body), nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if clear {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s answers from no collection by default\n", args[0])
|
||||
} else {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s answers from %s by default\n", args[0], strings.Join(args[1:], ", "))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
def.Flags().BoolVar(&clear, "clear", false, "no default any more")
|
||||
c.AddCommand(def)
|
||||
|
||||
var dsModel, dsOut string
|
||||
var dsPer, dsMax int
|
||||
dataset := &cobra.Command{
|
||||
Use: "dataset COLLECTION --model MODEL",
|
||||
Short: "Write training data from a collection: questions and answers, for an adapter",
|
||||
Long: "A chat model of the cluster reads each chunk of the collection and writes\n" +
|
||||
"questions a person might ask of it, with the answers it holds, one JSON object\n" +
|
||||
"a line ({\"q\", \"a\", \"doc\", \"ord\"}). Read it through and correct it before\n" +
|
||||
"training on it (llmgw ctl train): what the model got wrong, an adapter learns.\n\n" +
|
||||
"An adapter learns how answers sound — the organisation's words, format and\n" +
|
||||
"tone — far better than what they say; the facts stay the knowledge base's.",
|
||||
Example: " llmgw ctl kb dataset Acme/policies --model qwen2.5-coder-14b-instruct --out policies.jsonl",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
w := cmd.OutOrStdout()
|
||||
if dsOut != "" {
|
||||
f, err := os.Create(dsOut)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
w = f
|
||||
}
|
||||
lw := &datasetWriter{pairs: w, problems: cmd.ErrOrStderr()}
|
||||
body, _ := json.Marshal(adminapi.KBDatasetRequest{Model: dsModel, PerChunk: dsPer, MaxChunks: dsMax})
|
||||
err := client().patient(6*time.Hour).do(cmd.Context(), "POST",
|
||||
"/api/v1/kb/collections/"+url.PathEscape(args[0])+"/dataset", json.RawMessage(body), lw)
|
||||
lw.flush()
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), "%d pairs from %d chunks; %d chunks gave none\n", lw.n, len(lw.chunks), lw.failed)
|
||||
return err
|
||||
},
|
||||
}
|
||||
dataset.Flags().StringVar(&dsModel, "model", "", "the chat model that writes the questions and answers")
|
||||
dataset.Flags().StringVar(&dsOut, "out", "", "the file to write (default: standard output)")
|
||||
dataset.Flags().IntVar(&dsPer, "per-chunk", 8, "pairs from each chunk")
|
||||
dataset.Flags().IntVar(&dsMax, "max-chunks", 500, "the most chunks read")
|
||||
_ = dataset.MarkFlagRequired("model")
|
||||
c.AddCommand(dataset)
|
||||
|
||||
var k int
|
||||
search := &cobra.Command{
|
||||
Use: "search COLLECTION QUERY...",
|
||||
Short: "Try a search of a collection, as an administrator (no tenancy applies)",
|
||||
Args: cobra.MinimumNArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
body, _ := json.Marshal(adminapi.KBSearchRequest{Collection: args[0], Query: strings.Join(args[1:], " "), K: k})
|
||||
var hits []adminapi.KBHitView
|
||||
if err := client().do(cmd.Context(), "POST", "/api/v1/kb/search", json.RawMessage(body), &hits); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(hits)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
if len(hits) == 0 {
|
||||
fmt.Fprintln(out, "Nothing found.")
|
||||
}
|
||||
for i, h := range hits {
|
||||
fmt.Fprintf(out, "%d. %s #%d\n%s\n\n", i+1, h.Document, h.Ord, indent(strings.TrimSpace(h.Text), " "))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
search.Flags().IntVarP(&k, "k", "k", 5, "how many chunks")
|
||||
c.AddCommand(search)
|
||||
return c
|
||||
}
|
||||
|
||||
// datasetWriter splits the dataset stream: pairs to the file, problems to
|
||||
// the terminal, counted as they pass.
|
||||
type datasetWriter struct {
|
||||
pairs, problems io.Writer
|
||||
buf []byte
|
||||
n, failed int
|
||||
chunks map[string]bool
|
||||
}
|
||||
|
||||
func (d *datasetWriter) Write(p []byte) (int, error) {
|
||||
d.buf = append(d.buf, p...)
|
||||
for {
|
||||
i := bytes.IndexByte(d.buf, '\n')
|
||||
if i < 0 {
|
||||
return len(p), nil
|
||||
}
|
||||
d.line(d.buf[:i])
|
||||
d.buf = d.buf[i+1:]
|
||||
}
|
||||
}
|
||||
|
||||
func (d *datasetWriter) flush() {
|
||||
if len(bytes.TrimSpace(d.buf)) > 0 {
|
||||
d.line(d.buf)
|
||||
}
|
||||
d.buf = nil
|
||||
}
|
||||
|
||||
func (d *datasetWriter) line(b []byte) {
|
||||
var l adminapi.KBDatasetLine
|
||||
if json.Unmarshal(b, &l) != nil {
|
||||
return
|
||||
}
|
||||
if d.chunks == nil {
|
||||
d.chunks = map[string]bool{}
|
||||
}
|
||||
d.chunks[fmt.Sprintf("%s#%d", l.Doc, l.Ord)] = true
|
||||
if l.Error != "" {
|
||||
d.failed++
|
||||
fmt.Fprintf(d.problems, " %s #%d: %s\n", l.Doc, l.Ord, l.Error)
|
||||
return
|
||||
}
|
||||
d.n++
|
||||
out, _ := json.Marshal(l)
|
||||
fmt.Fprintln(d.pairs, string(out))
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -65,6 +66,14 @@ Safe to run again: it changes only what differs.`,
|
||||
}
|
||||
if !noFirewall {
|
||||
mirrorFirewall(cmd.Context(), cmd.OutOrStdout(), controlPlanes)
|
||||
// And after, as the cluster's settings say: the control plane
|
||||
// tells the mirror, which writes it beside its models.
|
||||
state := filepath.Join(*dir, mirror.FirewallFile)
|
||||
if err := installFirewallSync(cmd.Context(), firewall.Local{}, state); err != nil {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), " !! firewall: %v; the cluster's settings will not change it here\n", err)
|
||||
} else {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), " ok firewall: follows the cluster's settings ("+firewallSyncPathUnit+")")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
|
||||
+78
-30
@@ -3,6 +3,7 @@ package cli
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/user"
|
||||
"slices"
|
||||
@@ -32,7 +33,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
Use: "model",
|
||||
Short: "Add or remove models while the cluster is running",
|
||||
}
|
||||
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
c.AddCommand(newModelAddCmd(client), newModelPresetsCmd(client, asJSON), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
|
||||
newTargetCmd(client, asJSON), newAdaptersCmd(client),
|
||||
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
|
||||
@@ -46,49 +47,74 @@ func newModelAddCmd(client func() *ctlClient) *cobra.Command {
|
||||
var path string
|
||||
var skipCheck bool
|
||||
c := &cobra.Command{
|
||||
Use: "add -f <file.yaml>",
|
||||
Short: "Add a model from a definition file",
|
||||
Long: "The definition is the same YAML a model takes in server.yaml, so an\n" +
|
||||
Use: "add PRESET | -f <file.yaml>",
|
||||
Short: "Add a model that comes with llmgw, or one from a definition file",
|
||||
Long: "A preset is a model that comes with llmgw, its workflow, nodes and licence\n" +
|
||||
"already found and tried: " + strings.Join(catalog.PresetNames(), ", ") + " (llmgw ctl model presets).\n" +
|
||||
"Adding a ComfyUI one also has the nodes install ComfyUI, when the cluster\n" +
|
||||
"has not been told otherwise.\n\n" +
|
||||
"A definition is the same YAML a model takes in server.yaml, so an\n" +
|
||||
"entry can be moved between the two by copying it.\n\n" +
|
||||
"The catalog is rebuilt and validated before anything is written, so a\n" +
|
||||
"definition that collides with an existing name, or names a fallback that\n" +
|
||||
"does not exist, is refused rather than stored and then ignored on every\n" +
|
||||
"start. Once accepted it takes effect on the reconciler's next pass — no\n" +
|
||||
"restart — and survives one.",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if path == "" {
|
||||
return fmt.Errorf("--file is required: the model definition, as YAML")
|
||||
}
|
||||
// Read by koanf, the same parser and the same tags that read
|
||||
// server.yaml — so an entry can be moved between the two by copying
|
||||
// it, and a field that works in one cannot silently mean nothing in
|
||||
// the other.
|
||||
k := koanf.New(".")
|
||||
if err := k.Load(file.Provider(path), yaml.Parser()); err != nil {
|
||||
return fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
Example: " llmgw ctl model add triposr\n" +
|
||||
" llmgw ctl model add -f qwen.yaml",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var m catalog.Model
|
||||
if err := k.UnmarshalWithConf("", &m, koanf.UnmarshalConf{Tag: "koanf"}); err != nil {
|
||||
return fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
if m.Name == "" {
|
||||
return fmt.Errorf("%s: the definition has no name", path)
|
||||
switch {
|
||||
case len(args) == 1 && path != "":
|
||||
return fmt.Errorf("a preset or a file, not both")
|
||||
case len(args) == 1:
|
||||
p, ok := catalog.PresetNamed(args[0])
|
||||
if !ok {
|
||||
return fmt.Errorf("no preset %q: there are %s; any other model is added from a file (-f)",
|
||||
args[0], strings.Join(catalog.PresetNames(), ", "))
|
||||
}
|
||||
// The control plane adds it, and keeps that it was added as
|
||||
// a preset, by whom; one from before it could is sent the
|
||||
// entry, as it was then.
|
||||
var res adminapi.ModelAdded
|
||||
err := client().post(cmd.Context(), "/api/v1/presets/"+url.PathEscape(p.Model.Name), []byte("{}"), &res)
|
||||
if err == nil {
|
||||
fmt.Fprintf(cmd.OutOrStdout(),
|
||||
"%s added, and ComfyUI with it where the cluster had none. It is placed on the next\n"+
|
||||
"reconcile pass and loads when first asked for; watch it with:\n llmgw ctl models\n", p.Model.Name)
|
||||
return nil
|
||||
}
|
||||
if !olderControlPlane(err) {
|
||||
return err
|
||||
}
|
||||
m = p.Model
|
||||
case path == "":
|
||||
return fmt.Errorf("a preset (%s) or --file, the model definition as YAML",
|
||||
strings.Join(catalog.PresetNames(), ", "))
|
||||
default:
|
||||
var err error
|
||||
if m, err = readModelFile(path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Before the catalog is touched, because a source that does not
|
||||
// resolve is otherwise discovered by a replica failing to start
|
||||
// minutes later on a node — with an error naming the node rather
|
||||
// than the typo.
|
||||
// Every source the model's profiles load, not the model's alone.
|
||||
if !skipCheck {
|
||||
size, err := sourcecheck.Check(cmd.Context(), m.Source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w\n Pass --no-source-check to add it anyway, "+
|
||||
"which is right when the mirror is not up yet or the nodes "+
|
||||
"reach it by an address this machine does not.", err)
|
||||
}
|
||||
if _, isURL := sourcecheck.HTTPSource(m.Source); isURL {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), sourceNote(m.Source, size))
|
||||
for _, src := range m.WeightSources() {
|
||||
size, err := sourcecheck.Check(cmd.Context(), src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w\n Pass --no-source-check to add it anyway, "+
|
||||
"which is right when the mirror is not up yet or the nodes "+
|
||||
"reach it by an address this machine does not.", err)
|
||||
}
|
||||
if _, isURL := sourcecheck.HTTPSource(src); isURL {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), sourceNote(src, size))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -104,6 +130,9 @@ func newModelAddCmd(client func() *ctlClient) *cobra.Command {
|
||||
fmt.Fprintf(cmd.OutOrStdout(),
|
||||
"%s added. It will be placed on the next reconcile pass; watch it with:\n"+
|
||||
" llmgw ctl models\n", m.Name)
|
||||
if m.ServesComfyUI() && !m.IsExternal() {
|
||||
return ensureComfyUI(cmd, client, m.Name)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
@@ -113,6 +142,25 @@ func newModelAddCmd(client func() *ctlClient) *cobra.Command {
|
||||
return c
|
||||
}
|
||||
|
||||
// readModelFile reads a model's definition. By koanf, the same parser and
|
||||
// the same tags that read server.yaml — so an entry can be moved between the
|
||||
// two by copying it, and a field that works in one cannot silently mean
|
||||
// nothing in the other.
|
||||
func readModelFile(path string) (catalog.Model, error) {
|
||||
var m catalog.Model
|
||||
k := koanf.New(".")
|
||||
if err := k.Load(file.Provider(path), yaml.Parser()); err != nil {
|
||||
return m, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
if err := k.UnmarshalWithConf("", &m, koanf.UnmarshalConf{Tag: "koanf"}); err != nil {
|
||||
return m, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
if m.Name == "" {
|
||||
return m, fmt.Errorf("%s: the definition has no name", path)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func newModelRemoveCmd(client func() *ctlClient) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "remove <name>",
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
)
|
||||
|
||||
func newModelPresetsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
Use: "presets",
|
||||
Short: "The models that come with llmgw: whether each fits and is here",
|
||||
Long: "Each is a catalog entry whose workflow, custom nodes, packages and licence\n" +
|
||||
"were found and tried, and whose profile says the card it needs rather than\n" +
|
||||
"a class, so it places on any site's cards. Add one with\n" +
|
||||
"`llmgw ctl model add NAME`.\n\n" +
|
||||
"While presets.auto is on, the leader adds one by itself when a card fits it\n" +
|
||||
"and the catalog has nothing that makes what it makes; one somebody removed\n" +
|
||||
"is not added again by itself.",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
var v adminapi.PresetsView
|
||||
err := client().get(cmd.Context(), "/api/v1/presets", &v)
|
||||
if err != nil && !olderControlPlane(err) {
|
||||
return err
|
||||
}
|
||||
if *asJSON && err == nil {
|
||||
return emitJSON(v)
|
||||
}
|
||||
w := tabTo(cmd.OutOrStdout())
|
||||
if err != nil {
|
||||
// A control plane from before it kept them: what this
|
||||
// release has, with nothing about the cluster.
|
||||
fmt.Fprintln(w, "NAME\tMAKES\tCARD\tLICENCE\tWHAT IT IS FOR")
|
||||
for _, p := range catalog.Presets() {
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", p.Model.Name, p.Makes, presetCard(p.Model),
|
||||
presetLicence(p.Model), p.Model.Description)
|
||||
}
|
||||
return w.Flush()
|
||||
}
|
||||
fmt.Fprintln(w, "NAME\tMAKES\tCARD\tLICENCE\tFITS\tHERE\tWHAT IT IS FOR")
|
||||
for _, p := range v.Presets {
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", p.Name, p.Makes, p.Card, licenceText(p.Licence),
|
||||
orDash(strings.Join(p.FitsClasses, ",")), presetState(p), p.Description)
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if v.Auto {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "\nAdded by themselves where a card fits (presets.auto); "+
|
||||
"llmgw ctl model presets auto off stops it.")
|
||||
} else {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "\nNot added by themselves; llmgw ctl model presets auto on starts it.")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.AddCommand(&cobra.Command{
|
||||
Use: "auto on|off",
|
||||
Short: "Whether the leader adds a preset by itself where a card fits it",
|
||||
Args: cobra.ExactArgs(1),
|
||||
ValidArgs: []string{"on", "off"},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if args[0] != "on" && args[0] != "off" {
|
||||
return fmt.Errorf("on or off, not %q", args[0])
|
||||
}
|
||||
body, _ := json.Marshal(adminapi.PresetsAutoRequest{On: args[0] == "on"})
|
||||
var v adminapi.PresetsView
|
||||
if err := client().put(cmd.Context(), "/api/v1/presets/auto", body, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
if v.Auto {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "Presets are added by themselves where a card fits.")
|
||||
} else {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "Presets are no longer added by themselves.")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
return c
|
||||
}
|
||||
|
||||
// olderControlPlane is an error from a control plane that has no such path:
|
||||
// one from before it was added.
|
||||
func olderControlPlane(err error) bool {
|
||||
s := err.Error()
|
||||
return strings.Contains(s, "answered with a page") || strings.HasPrefix(s, "404 Not Found: 404 page not found")
|
||||
}
|
||||
|
||||
func presetState(p adminapi.PresetView) string {
|
||||
switch {
|
||||
case p.InCatalog && p.AddedBy == "auto":
|
||||
return "added by itself"
|
||||
case p.InCatalog && p.AddedBy != "":
|
||||
return "added by " + p.AddedBy
|
||||
case p.InCatalog:
|
||||
return "here"
|
||||
case p.Declined:
|
||||
return "removed; stays out"
|
||||
}
|
||||
return "—"
|
||||
}
|
||||
|
||||
func licenceText(l catalog.License) string {
|
||||
if l.Commercial != nil && !*l.Commercial {
|
||||
return l.Name + ", not commercial"
|
||||
}
|
||||
return l.Name
|
||||
}
|
||||
|
||||
func presetCard(m catalog.Model) string {
|
||||
if len(m.Profiles) == 0 || m.Profiles[0].GPU == nil {
|
||||
return "—"
|
||||
}
|
||||
return m.Profiles[0].GPU.String()
|
||||
}
|
||||
|
||||
func presetLicence(m catalog.Model) string {
|
||||
if m.License == nil {
|
||||
return "—"
|
||||
}
|
||||
l := m.License.Name
|
||||
if c := m.License.Commercial; c != nil && !*c {
|
||||
l += ", not commercial"
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// ensureComfyUI has the nodes install ComfyUI for a model that needs it,
|
||||
// unless the cluster names a version already, or none on purpose.
|
||||
func ensureComfyUI(cmd *cobra.Command, client func() *ctlClient, model string) error {
|
||||
out := cmd.OutOrStdout()
|
||||
path := []string{"engines", catalog.EngineComfyUI}
|
||||
var cur adminapi.SettingsView
|
||||
if err := client().get(cmd.Context(), "/api/v1/settings", &cur); err != nil {
|
||||
return err
|
||||
}
|
||||
doc, err := settings.FromBytes("settings", []byte(cur.Body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
version := provision.ComfyUIVersion()
|
||||
if n := doc.Node(path); n != nil {
|
||||
if n.Value == "" {
|
||||
fmt.Fprintf(out, "engines.comfyui is set to none, so the nodes install no ComfyUI for it: set a version "+
|
||||
"(llmgw ctl config set engines.comfyui %s) or name one installed by hand in agent.yaml\n", version)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintf(out, "Nodes with an NVIDIA GPU install ComfyUI %s for it, about 10 GB each:\n", version)
|
||||
if err := doc.SetYAML(path, version); err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := doc.Bytes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return putSettings(cmd, client, cur, b, "ComfyUI for "+model)
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
// presetAdmin is an admin API holding settings, which records the models
|
||||
// added and the settings put.
|
||||
type presetAdmin struct {
|
||||
// api is a control plane that adds presets itself; without it, one
|
||||
// from before, which answers their paths with a 404.
|
||||
api bool
|
||||
byAPI []string
|
||||
mu sync.Mutex
|
||||
settings string
|
||||
added []catalog.Model
|
||||
put []string
|
||||
}
|
||||
|
||||
func (a *presetAdmin) client(t *testing.T) func() *ctlClient {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch {
|
||||
case a.api && r.Method == http.MethodPost && strings.HasPrefix(r.URL.Path, "/api/v1/presets/"):
|
||||
name := strings.TrimPrefix(r.URL.Path, "/api/v1/presets/")
|
||||
a.byAPI = append(a.byAPI, name)
|
||||
_ = json.NewEncoder(w).Encode(adminapi.ModelAdded{Added: name})
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/api/v1/models":
|
||||
var m catalog.Model
|
||||
_ = json.Unmarshal(body, &m)
|
||||
a.added = append(a.added, m)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"name": m.Name})
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/settings":
|
||||
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 1, Body: a.settings})
|
||||
case r.Method == http.MethodPut && r.URL.Path == "/api/v1/settings":
|
||||
var req adminapi.SettingsRequest
|
||||
_ = json.Unmarshal(body, &req)
|
||||
a.put = append(a.put, req.Body)
|
||||
a.settings = req.Body
|
||||
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 2, Body: req.Body})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return func() *ctlClient { return newCtlClient(strings.TrimPrefix(srv.URL, "http://"), "") }
|
||||
}
|
||||
|
||||
func runModelAdd(t *testing.T, client func() *ctlClient, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
cmd := newModelAddCmd(client)
|
||||
var out bytes.Buffer
|
||||
cmd.SetOut(&out)
|
||||
cmd.SetErr(&out)
|
||||
cmd.SetArgs(append(args, "--no-source-check"))
|
||||
err := cmd.ExecuteContext(t.Context())
|
||||
return out.String(), err
|
||||
}
|
||||
|
||||
func TestAPresetIsAddedByTheControlPlane(t *testing.T) {
|
||||
a := &presetAdmin{api: true}
|
||||
out, err := runModelAdd(t, a.client(t), "triposr")
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if len(a.byAPI) != 1 || a.byAPI[0] != "triposr" || len(a.added) != 0 || len(a.put) != 0 {
|
||||
t.Errorf("by the API %v; sent the entry %v; settings %v", a.byAPI, a.added, a.put)
|
||||
}
|
||||
}
|
||||
|
||||
// A control plane from before presets were its own is sent the entry, and
|
||||
// ComfyUI set beside it, as the release that brought presets did.
|
||||
func TestAPresetIsAddedByNameAndHasTheNodesInstallComfyUI(t *testing.T) {
|
||||
a := &presetAdmin{settings: "router:\n retries: 2\n"}
|
||||
out, err := runModelAdd(t, a.client(t), "triposr")
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if len(a.added) != 1 || a.added[0].Name != "triposr" || a.added[0].ComfyUI.Workflow != "triposr" {
|
||||
t.Fatalf("added %+v", a.added)
|
||||
}
|
||||
if len(a.put) != 1 || !strings.Contains(a.put[0], "comfyui: 0.39.2") || !strings.Contains(a.put[0], "retries: 2") {
|
||||
t.Fatalf("settings put %q", a.put)
|
||||
}
|
||||
if !strings.Contains(out, "install ComfyUI 0.39.2") {
|
||||
t.Errorf("not said:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAClusterThatNamesItsComfyUIKeepsIt(t *testing.T) {
|
||||
for _, settings := range []string{"engines:\n comfyui: 0.38.0\n", "engines:\n comfyui: \"\"\n"} {
|
||||
a := &presetAdmin{settings: settings}
|
||||
out, err := runModelAdd(t, a.client(t), "flux-schnell")
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if len(a.put) != 0 {
|
||||
t.Errorf("%q: settings changed to %q", settings, a.put)
|
||||
}
|
||||
if none := strings.Contains(settings, `""`); none != strings.Contains(out, "set to none") {
|
||||
t.Errorf("%q: %s", settings, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnknownPresetNamesTheOnesThereAre(t *testing.T) {
|
||||
a := &presetAdmin{}
|
||||
_, err := runModelAdd(t, a.client(t), "sdxl")
|
||||
if err == nil || !strings.Contains(err.Error(), "flux-schnell, triposr") || len(a.added) != 0 {
|
||||
t.Errorf("%v, added %v", err, a.added)
|
||||
}
|
||||
if _, err := runModelAdd(t, a.client(t), "triposr", "-f", "x.yaml"); err == nil {
|
||||
t.Error("a preset and a file were both taken")
|
||||
}
|
||||
}
|
||||
@@ -60,6 +60,8 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default: the one that suits the cluster best)")
|
||||
f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant")
|
||||
f.StringVar(&req.Projector, "projector", "", "a vision model's mmproj file, fetched and loaded beside it")
|
||||
f.BoolVar(&req.Embeddings, "embeddings", false, "serve it as an embedding model (its GGUF header says so for most): "+
|
||||
"--embedding, 32 slots of 1024 tokens unless --ctx-pool or --context-length says otherwise")
|
||||
f.StringVar(&req.Format, "format", "gguf", "gguf, run by llama.cpp, or safetensors: the original weights, run by vLLM on GPUs")
|
||||
f.BoolVar(&req.CPU, "cpu", false, "run it on the CPU, whatever the plan says")
|
||||
f.StringVar(&req.GPUClass, "gpu-class", "", "run it on GPUs of this class, whatever the plan says")
|
||||
|
||||
@@ -54,6 +54,7 @@ func Root() *cobra.Command {
|
||||
newMirrorCmd(),
|
||||
newBundleCmd(),
|
||||
newSimEngineCmd(),
|
||||
newComfyUIGateCmd(),
|
||||
newDoctorCmd(),
|
||||
newNodeCmd(),
|
||||
newSelfUpgradeCmd(),
|
||||
@@ -63,6 +64,10 @@ func Root() *cobra.Command {
|
||||
newWipeCmd(),
|
||||
newDocsCmd(),
|
||||
)
|
||||
// status, start, stop and restart: the whole system; service: this
|
||||
// machine's services.
|
||||
root.AddCommand(newSystemCmds()...)
|
||||
root.AddCommand(newServiceCmd())
|
||||
return root
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
@@ -70,6 +73,9 @@ func newSelfUpgradeCmd() *cobra.Command {
|
||||
if strings.TrimSpace(string(b)) == "active" {
|
||||
if steady++; steady >= 3 {
|
||||
fmt.Fprintf(out, "%s restarted and running\n", service)
|
||||
if service == "llmgw-server" {
|
||||
nodesLeftBehind(cmd.Context(), out, newCtlClient("127.0.0.1:9090"), v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
@@ -85,3 +91,25 @@ func newSelfUpgradeCmd() *cobra.Command {
|
||||
c.Flags().StringVar(&service, "service", "", "the systemd unit to restart afterwards, e.g. llmgw-mirror")
|
||||
return c
|
||||
}
|
||||
|
||||
// nodesLeftBehind says, after a control plane alone was upgraded, that its
|
||||
// nodes were not, and how they are: this command looks like the cluster's
|
||||
// upgrade and is not, and a site's nodes stayed a release behind after it.
|
||||
func nodesLeftBehind(ctx context.Context, out io.Writer, c *ctlClient, v string) {
|
||||
var nodes []adminapi.NodeView
|
||||
_ = c.patient(20*time.Second).get(ctx, "/api/v1/nodes", &nodes)
|
||||
behind := 0
|
||||
for _, n := range nodes {
|
||||
if n.AgentVersion != v {
|
||||
behind++
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(nodes) == 0:
|
||||
fmt.Fprintf(out, "This upgraded this machine only. The nodes are upgraded from the leading "+
|
||||
"control plane with:\n sudo llmgw ctl upgrade %s --skip-control-plane\n", v)
|
||||
case behind > 0:
|
||||
fmt.Fprintf(out, "This upgraded this machine only: %d of %d node(s) are not on %s. On the leading "+
|
||||
"control plane, run:\n sudo llmgw ctl upgrade %s --skip-control-plane\n", behind, len(nodes), v, v)
|
||||
}
|
||||
}
|
||||
+178
-10
@@ -5,6 +5,7 @@ import (
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
@@ -48,6 +49,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fleetmetrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/train"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
|
||||
@@ -186,6 +188,7 @@ func newServerCmd() *cobra.Command {
|
||||
|
||||
rt.Start(ctx)
|
||||
defer rt.Stop()
|
||||
rt.SetStopped(cfg.Serving.Stopped)
|
||||
|
||||
// Which cluster this is, to federated peers: the id in the hop
|
||||
// header that stops a request going round two clusters that
|
||||
@@ -207,6 +210,10 @@ func newServerCmd() *cobra.Command {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if st := rt.Store(); st != nil {
|
||||
// Each control plane raises the same events; one sends each.
|
||||
notifier.SetClaimer(st)
|
||||
}
|
||||
rt.SetNotifier(notifier)
|
||||
if gate != nil && notifier != nil {
|
||||
gate.SetJudgeAlarm(func(model string, err error) {
|
||||
@@ -230,6 +237,9 @@ func newServerCmd() *cobra.Command {
|
||||
var agentHandler http.Handler
|
||||
var agentPath string
|
||||
var admin *control.Controller
|
||||
// What the settings say of the firewall, for the mirror and the
|
||||
// store's members; nil without a control plane.
|
||||
var fwPub *firewallPublisher
|
||||
updateMetrics := &modelserve.UpdateCollector{}
|
||||
// The store's cluster, when it is one; read by the metrics and
|
||||
// by `ctl status`.
|
||||
@@ -252,6 +262,9 @@ func newServerCmd() *cobra.Command {
|
||||
}
|
||||
defer ctl.Close()
|
||||
agentSvc = svc
|
||||
// Stopped by the operators (llmgw stop): nothing placed.
|
||||
ctl.Reconciler().SetStopped(cfg.Serving.Stopped)
|
||||
ctl.Reconciler().SetEvictionTimes(cfg.OnDemand.EvictAfterIdle, cfg.OnDemand.MinLifetime)
|
||||
// Whether nodes keep llmgw's firewall, as the settings say now.
|
||||
svc.SetFirewall(func() *llmgwv1.Firewall {
|
||||
c := current.Load()
|
||||
@@ -259,9 +272,31 @@ func newServerCmd() *cobra.Command {
|
||||
return &llmgwv1.Firewall{Enabled: c.Firewall.Enabled, Networks: c.FirewallNetworks,
|
||||
EnginePortsFrom: int32(from), EnginePortsTo: int32(to)} //nolint:gosec // ports
|
||||
})
|
||||
// What each node's agent runs on, as the settings say now.
|
||||
svc.SetAgentSettings(func(node string) string {
|
||||
b, err := config.Marshal(current.Load().Agents.For(node))
|
||||
if err != nil {
|
||||
log.Warn("could not write a node's agent settings", "node", node, "err", err)
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
})
|
||||
ctl.Reconciler().SetNotifier(notifier)
|
||||
admin = ctl
|
||||
agentPath, agentHandler = svc.Handler()
|
||||
fwPub = newFirewallPublisher(func() config.Server { return *current.Load() }, rt.Store(),
|
||||
func() []string {
|
||||
var out []string
|
||||
for _, n := range ctl.Inventory().Nodes() {
|
||||
out = append(out, n.MgmtAddr)
|
||||
}
|
||||
return out
|
||||
},
|
||||
func() bool { return ctl.Leadership() == nil || ctl.Leadership().IsLeader() }, log)
|
||||
go fwPub.run(ctx, 5*time.Minute)
|
||||
if l := ctl.Leadership(); l != nil {
|
||||
l.OnLead(fwPub.Kick)
|
||||
}
|
||||
|
||||
// Before AdminHandler, which is where the registry is built —
|
||||
// registering after it silently publishes nothing, which is
|
||||
@@ -367,11 +402,26 @@ func newServerCmd() *cobra.Command {
|
||||
if st := rt.Store(); st != nil {
|
||||
// Removing a node revokes its certificate.
|
||||
api.SetRevoker(st.RevokeNode)
|
||||
if agentSvc != nil {
|
||||
api.SetAgentRestarter(agentSvc.RestartAgents)
|
||||
}
|
||||
api.SetAudit(st)
|
||||
// Knowledge bases, with the cluster's own models to embed a
|
||||
// probe (a collection's dimension) and an operator's search.
|
||||
api.SetKB(&adminapi.KBAdmin{Store: st, Embed: embedder(rt.Replay)})
|
||||
// LoRA adapters trained on the nodes.
|
||||
if agentSvc != nil {
|
||||
trainer := train.New(st, agentSvc, log)
|
||||
agentSvc.SetTrain(trainer)
|
||||
go trainer.Run(ctx)
|
||||
api.SetTrain(&adminapi.TrainAdmin{Store: st, Service: trainer})
|
||||
}
|
||||
// The cluster's settings, each change checked against this
|
||||
// machine's own keys before it is kept.
|
||||
api.SetSettings(&adminapi.Settings{Store: st, LocalKeys: config.LocalKeys(),
|
||||
Check: func(body []byte) error { return checkClusterSettings(configPath, body) }})
|
||||
// And the same document on the portal, for administrators.
|
||||
rt.SetSettingsCheck(func(body []byte) error { return checkClusterSettings(configPath, body) }, config.LocalKeys())
|
||||
}
|
||||
// Catalog changes need somewhere to persist, so this is wired
|
||||
// only when there is a store. Without one, the endpoints say so
|
||||
@@ -416,6 +466,14 @@ func newServerCmd() *cobra.Command {
|
||||
}
|
||||
api.SetModels(svc)
|
||||
catalogSvc = svc
|
||||
// The models that come with llmgw: added by the leader
|
||||
// when a node fits one and the catalog makes nothing of
|
||||
// its kind, unless somebody removed it or turned it off.
|
||||
api.SetPresets(st, func() adminapi.PresetConfig {
|
||||
c := current.Load()
|
||||
return adminapi.PresetConfig{Auto: c.Presets.Auto, ComfyUI: c.Engines[catalog.EngineComfyUI]}
|
||||
})
|
||||
go runPresetsPass(ctx, api, admin.Leadership(), log)
|
||||
// And the mirror page, so a model that has just been
|
||||
// fetched can be served without writing YAML by hand.
|
||||
rt.SetCatalogAdmin(svc)
|
||||
@@ -428,6 +486,7 @@ func newServerCmd() *cobra.Command {
|
||||
api.SetUsage(st)
|
||||
// Capacity quotas, set and read over this API.
|
||||
api.SetQuotas(st)
|
||||
api.SetTenancy(st)
|
||||
}
|
||||
api.Register(adminMux)
|
||||
|
||||
@@ -489,6 +548,16 @@ func newServerCmd() *cobra.Command {
|
||||
return fmt.Errorf("the model mirror: %w", err)
|
||||
}
|
||||
rt.SetMirror(mc)
|
||||
if fwPub != nil {
|
||||
fwPub.setMirror(mc)
|
||||
}
|
||||
// Adapters trained on the nodes are kept there too; sending one
|
||||
// takes longer than the mirror's usual answers.
|
||||
if adminAPI != nil {
|
||||
if up, err := mirror.NewClient(cfg.Mirror.API, cfg.Mirror.TLS, 10*time.Minute); err == nil {
|
||||
adminAPI.SetTrainMirror(up)
|
||||
}
|
||||
}
|
||||
// Browsing the registry is what turns "paste a resolve URL"
|
||||
// into "find the model". It needs a way out to the internet
|
||||
// from this machine, which the mirror's own fetching does not.
|
||||
@@ -512,6 +581,21 @@ func newServerCmd() *cobra.Command {
|
||||
})
|
||||
go serve.WatchCanaries(ctx, 30*time.Second)
|
||||
adminAPI.SetServe(serve)
|
||||
if gate != nil {
|
||||
// A judge a policy names and the catalog lacks is
|
||||
// fetched and served, on the leader.
|
||||
keeper := &judgeKeeper{
|
||||
library: gate.Library,
|
||||
models: func() []catalog.Model { return admin.Catalog().Models() },
|
||||
serve: serve,
|
||||
leader: func() bool {
|
||||
return admin.Leadership() == nil || admin.Leadership().IsLeader()
|
||||
},
|
||||
gpus: func() []adminapi.ClassUse { return adminAPI.Summary().GPUs },
|
||||
log: log,
|
||||
}
|
||||
go keeper.run(ctx, 30*time.Second)
|
||||
}
|
||||
updateMetrics.Set(serve)
|
||||
go serve.WatchUpdates(ctx, cfg.Mirror.UpdateCheck)
|
||||
}
|
||||
@@ -676,6 +760,14 @@ func newServerCmd() *cobra.Command {
|
||||
return false
|
||||
}
|
||||
current.Store(&next)
|
||||
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "serving") }) {
|
||||
rt.SetStopped(next.Serving.Stopped)
|
||||
if admin != nil {
|
||||
admin.Reconciler().SetStopped(next.Serving.Stopped)
|
||||
}
|
||||
log.Warn("the service is " + map[bool]string{true: "stopped", false: "started"}[next.Serving.Stopped] +
|
||||
" by the cluster's settings")
|
||||
}
|
||||
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "firewall") }) {
|
||||
if err := writeFirewallState(next); err != nil {
|
||||
log.Warn("could not record the firewall's settings for this machine", "err", err)
|
||||
@@ -683,6 +775,16 @@ func newServerCmd() *cobra.Command {
|
||||
if agentSvc != nil {
|
||||
agentSvc.BroadcastFirewall()
|
||||
}
|
||||
if fwPub != nil {
|
||||
fwPub.Kick()
|
||||
}
|
||||
}
|
||||
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "on_demand") }) && admin != nil {
|
||||
admin.Reconciler().SetEvictionTimes(next.OnDemand.EvictAfterIdle, next.OnDemand.MinLifetime)
|
||||
}
|
||||
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "agents") }) && agentSvc != nil {
|
||||
// A moment apart per node, so off this goroutine.
|
||||
go agentSvc.BroadcastAgentSettings()
|
||||
}
|
||||
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "guard") }) {
|
||||
if gate == nil {
|
||||
@@ -1083,18 +1185,18 @@ func servableCheck(llamaVersion string) func(context.Context, catalog.Model) err
|
||||
if llamaVersion == "" {
|
||||
return nil
|
||||
}
|
||||
usesLlama := false
|
||||
// Each source a llama.cpp profile loads: its own, or the model's.
|
||||
for _, p := range m.Profiles {
|
||||
if p.Engine == "llamacpp" {
|
||||
usesLlama = true
|
||||
if p.Engine != "llamacpp" {
|
||||
continue
|
||||
}
|
||||
url, ok := sourcecheck.HTTPSource(m.SourceOf(p))
|
||||
if !ok || !gguf.IsGGUF(url) {
|
||||
continue
|
||||
}
|
||||
if v := gguf.Check(ctx, url, llamaVersion); !v.Servable {
|
||||
return fmt.Errorf("%s cannot be served: %s", m.Name, v.Reason)
|
||||
}
|
||||
}
|
||||
url, ok := sourcecheck.HTTPSource(m.Source)
|
||||
if !usesLlama || !ok || !gguf.IsGGUF(url) {
|
||||
return nil
|
||||
}
|
||||
if v := gguf.Check(ctx, url, llamaVersion); !v.Servable {
|
||||
return fmt.Errorf("%s cannot be served: %s", m.Name, v.Reason)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1154,3 +1256,69 @@ func federationID(ctx context.Context, st *storepkg.DB) (string, error) {
|
||||
// more of a secret than needed.
|
||||
return secret[:16], nil
|
||||
}
|
||||
|
||||
// embedder has a model embed one text through the gateway itself, the way a
|
||||
// kept request is replayed: for the administrator's knowledge-base
|
||||
// endpoints, not for anybody's traffic.
|
||||
func embedder(replay func(ctx context.Context, path, model string, body []byte) (int, []byte, time.Duration, error)) func(context.Context, string, string) ([]float32, error) {
|
||||
return func(ctx context.Context, model, text string) ([]float32, error) {
|
||||
body, _ := json.Marshal(map[string]any{"model": model, "input": []string{text}})
|
||||
status, out, _, err := replay(ctx, "/v1/embeddings", model, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if status != http.StatusOK {
|
||||
msg := strings.TrimSpace(string(out))
|
||||
var e struct {
|
||||
Error struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(out, &e) == nil && e.Error.Message != "" {
|
||||
msg = e.Error.Message
|
||||
}
|
||||
return nil, fmt.Errorf("%d: %s", status, msg)
|
||||
}
|
||||
var res struct {
|
||||
Data []struct {
|
||||
Embedding []float32 `json:"embedding"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(out, &res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(res.Data) != 1 || len(res.Data[0].Embedding) == 0 {
|
||||
return nil, fmt.Errorf("%s answered no embedding", model)
|
||||
}
|
||||
return res.Data[0].Embedding, nil
|
||||
}
|
||||
}
|
||||
|
||||
// presetsEvery is how often the leader looks for presets to add; the first
|
||||
// look waits as long, for the nodes to have registered their cards.
|
||||
const presetsEvery = time.Minute
|
||||
|
||||
// runPresetsPass has the leader add the presets a node fits and the catalog
|
||||
// lacks, now and then.
|
||||
func runPresetsPass(ctx context.Context, api *adminapi.API, lead *control.Leadership, log *slog.Logger) {
|
||||
t := time.NewTicker(presetsEvery)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
if lead != nil && !lead.IsLeader() {
|
||||
continue
|
||||
}
|
||||
added, err := api.PresetsPass(ctx)
|
||||
for _, name := range added {
|
||||
log.Info("added a preset by itself: a node fits it and the catalog made nothing of its kind",
|
||||
"model", name, "off_switch", "presets.auto")
|
||||
}
|
||||
if err != nil {
|
||||
log.Warn("could not add a preset", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -160,7 +160,7 @@ func (sa *serverAdd) firewall(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
// What applies the cluster's firewall setting there when it changes.
|
||||
if err := installFirewallSync(ctx, sa.remote); err != nil {
|
||||
if err := installFirewallSync(ctx, sa.remote, firewallStateFile); err != nil {
|
||||
fmt.Fprintf(sa.out, " !! firewall: %v\n", err)
|
||||
}
|
||||
if !cfg.Firewall.Enabled {
|
||||
|
||||
@@ -73,6 +73,9 @@ type serverInit struct {
|
||||
// patroni are user@host of the machines to keep the store on under
|
||||
// Patroni, replicated; this machine is the third etcd vote for two.
|
||||
patroni []string
|
||||
// webhooks are where the cluster's alerts go: put into its settings
|
||||
// once it runs, and each sent a test.
|
||||
webhooks []string
|
||||
// patroniNodes are the same machines already connected to, which the
|
||||
// setup screen does with the password it was given; when set, patroni
|
||||
// only names them.
|
||||
@@ -182,6 +185,9 @@ func newServerInitCmd() *cobra.Command {
|
||||
"leave the firewall alone, instead of keeping llmgw's internal ports to the cluster's networks")
|
||||
f.BoolVar(&in.postgres, "postgres", false,
|
||||
"keep the store in PostgreSQL on this machine, installing it and moving an existing SQLite store into it")
|
||||
f.StringArrayVar(&in.webhooks, "webhook", nil,
|
||||
"where alerts go — a model parked, a node lost, a budget spent: an http(s) address, Slack's, Mattermost's "+
|
||||
"and Rocket.Chat's sent Slack's text, any other the event as JSON (slack: or json: in front says which); repeatable")
|
||||
f.StringArrayVar(&in.patroni, "patroni", nil,
|
||||
"user@host of a machine to keep the store on, replicated under Patroni; give two or more. "+
|
||||
"Installs PostgreSQL, Patroni and etcd there over ssh, and moves the existing store in")
|
||||
@@ -249,6 +255,9 @@ func (in *serverInit) run(ctx context.Context) error {
|
||||
fmt.Fprintf(in.out, " !! firewall: %v; llmgw's ports answer every network this machine is on\n", err)
|
||||
}
|
||||
}
|
||||
if err := in.ensureWebhooks(ctx); err != nil {
|
||||
fmt.Fprintf(in.out, " !! webhooks: %v\n", err)
|
||||
}
|
||||
if in.judgeWanted() && in.configJudgesWith(guardModelName) {
|
||||
// Not fatal: without its judge the baseline policy still checks for
|
||||
// secrets and injection, and a judged request is served flagged.
|
||||
@@ -293,7 +302,7 @@ func (in *serverInit) configJudgesWith(model string) bool {
|
||||
return false
|
||||
}
|
||||
for _, p := range cfg.Guard.Policies {
|
||||
if p.Judge != nil && p.Judge.Model == model {
|
||||
if slices.Contains(p.Judge.Judges(), model) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -308,18 +317,21 @@ func guardSection(judge bool) string {
|
||||
# judge: {model: ` + guardModelName + `, input: block, output: block, ...}
|
||||
`
|
||||
if judge {
|
||||
j = ` # A judge model reads what patterns cannot: violence, weapons, abuse,
|
||||
# self-harm. Two replicas run on CPUs with AVX2, about half a second
|
||||
# for a short text; an answer is read every 600 characters as it
|
||||
# streams. With fail: closed, a request the judge cannot read within
|
||||
j = ` # A judge model reads what patterns cannot: violence, crime, sexual
|
||||
# content, self-harm, jailbreaks. Two replicas run on CPUs, about two
|
||||
# seconds for a short text; an answer is read every 600 characters as
|
||||
# it streams. With fail: closed, a request no judge can read within
|
||||
# latency_budget is refused (503); open serves it, flagged, unjudged.
|
||||
# Defamation (S5), specialized advice (S6), intellectual property (S8)
|
||||
# and elections (S13) are left out: they refuse too much ordinary work.
|
||||
# PII is the patterns' (above), and copyright, unethical acts and
|
||||
# politically sensitive topics are left out: they refuse too much
|
||||
# ordinary work. Controversial counts as safe; controversial: unsafe
|
||||
# makes it count. With a GPU: llmgw ctl guard judge enable --gpu, then
|
||||
# llmgw ctl guard judge use ` + defaultGPUJudge + ` ` + guardModelName + `.
|
||||
judge:
|
||||
model: ` + guardModelName + `
|
||||
models: [` + guardModelName + `]
|
||||
input: block
|
||||
output: block
|
||||
categories: [S1, S2, S3, S4, S7, S9, S10, S11, S12]
|
||||
categories: [Violent, Non-violent Illegal Acts, Sexual Content or Sexual Acts, Suicide & Self-Harm, Jailbreak]
|
||||
latency_budget: 5s
|
||||
`
|
||||
}
|
||||
|
||||
@@ -41,11 +41,13 @@ const (
|
||||
starterSource = "hf:Qwen/Qwen2.5-0.5B-Instruct-GGUF/qwen2.5-0.5b-instruct-q4_k_m.gguf"
|
||||
starterVersion = "starter"
|
||||
|
||||
// The guardrails' judge: Meta's Llama Guard 3 1B, which reads a prompt
|
||||
// or an answer and says whether it is unsafe and why. Q4_K_M is about
|
||||
// 900 MB and judges a short text in about half a second on a CPU.
|
||||
guardModelName = "llama-guard-3-1b"
|
||||
guardModelSource = "hf:QuantFactory/Llama-Guard-3-1B-GGUF/Llama-Guard-3-1B.Q4_K_M.gguf"
|
||||
// The guardrails' judge: Qwen3Guard-Gen 0.6B, which reads a prompt or an
|
||||
// answer and says whether it is safe, controversial or unsafe, and why.
|
||||
// Q8_0 is 0.8 GB and judges a short text in about two seconds on two CPU
|
||||
// threads. It replaced Llama Guard 3 1B, which refused about one in two
|
||||
// ordinary coding requests ("continue").
|
||||
guardModelName = defaultJudge
|
||||
guardModelSource = "hf:mradermacher/Qwen3Guard-Gen-0.6B-GGUF/Qwen3Guard-Gen-0.6B.Q8_0.gguf"
|
||||
guardModelVersion = "guard"
|
||||
|
||||
// localLabel marks the control plane's own node, so the starter model is
|
||||
@@ -443,12 +445,6 @@ func starterModel(source string, gpuClasses []string, pinned bool) catalog.Model
|
||||
}
|
||||
}
|
||||
|
||||
// llamaGuard3Categories name Llama Guard 3's S1 to S13, so a verdict is
|
||||
// recorded as "Violent Crimes" rather than "S1".
|
||||
var llamaGuard3Categories = []string{"Violent Crimes", "Non-Violent Crimes", "Sex-Related Crimes",
|
||||
"Child Sexual Exploitation", "Defamation", "Specialized Advice", "Privacy", "Intellectual Property",
|
||||
"Indiscriminate Weapons", "Hate", "Suicide & Self-Harm", "Sexual Content", "Elections"}
|
||||
|
||||
// guardModel is the judge's catalog entry: CPU only, so it takes no card from
|
||||
// the models people use, on whichever node has room.
|
||||
func guardModel(source string) catalog.Model {
|
||||
@@ -457,7 +453,7 @@ func guardModel(source string) catalog.Model {
|
||||
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
|
||||
Source: source,
|
||||
Role: catalog.RoleGuard,
|
||||
Guard: &catalog.GuardModel{Format: "llamaguard3", Categories: llamaGuard3Categories},
|
||||
Guard: &catalog.GuardModel{Format: "qwen3guard"},
|
||||
// Two: every judged request waits on it, and with fail: closed a
|
||||
// judge that is not there refuses everything.
|
||||
Scaling: catalog.Scaling{Min: 2, Max: 2},
|
||||
|
||||
@@ -7,12 +7,14 @@ import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
func TestTheInitAgentUnitIsTheShippedUnit(t *testing.T) {
|
||||
@@ -150,9 +152,17 @@ func TestTheInstalledConfigurationIsHardenedAndLoads(t *testing.T) {
|
||||
if (p.Judge != nil) != judge {
|
||||
t.Errorf("judge=%v: the policy's judge is %+v", judge, p.Judge)
|
||||
}
|
||||
if judge && (p.Judge.Model != guardModelName || p.Judge.Input != "block") {
|
||||
if judge && (!slices.Equal(p.Judge.Judges(), []string{guardModelName}) || p.Judge.Input != "block") {
|
||||
t.Errorf("judge %+v", p.Judge)
|
||||
}
|
||||
// A category the judge never names would never count.
|
||||
if judge {
|
||||
for _, c := range p.Judge.Categories {
|
||||
if !slices.Contains(guard.DefaultCategories("qwen3guard"), c) {
|
||||
t.Errorf("the baseline counts %q, which Qwen3Guard does not name", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
if cfg.Auth.MaxKeyTTL != 90*24*time.Hour || cfg.Auth.SignupOpen || cfg.DebugPprof || cfg.AdminRemote {
|
||||
t.Errorf("not hardened: ttl %v signup %v pprof %v remote %v",
|
||||
cfg.Auth.MaxKeyTTL, cfg.Auth.SignupOpen, cfg.DebugPprof, cfg.AdminRemote)
|
||||
@@ -161,13 +171,13 @@ func TestTheInstalledConfigurationIsHardenedAndLoads(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTheGuardModelIsAValidJudge(t *testing.T) {
|
||||
m := guardModel("http://10.0.0.5:8081/llama-guard-3-1b/guard/g.gguf")
|
||||
m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf")
|
||||
c, err := catalog.New([]catalog.Model{m})
|
||||
if err != nil {
|
||||
t.Fatalf("not a valid catalog entry: %v", err)
|
||||
}
|
||||
got, _ := c.Resolve(guardModelName)
|
||||
if !got.IsGuard() || got.Guard.Format != "llamaguard3" || len(got.Guard.Categories) != 13 || got.Scaling.Min != 2 {
|
||||
if !got.IsGuard() || got.Guard.Format != "qwen3guard" || got.Scaling.Min != 2 {
|
||||
t.Errorf("%+v", got)
|
||||
}
|
||||
for _, p := range got.Profiles {
|
||||
|
||||
@@ -93,6 +93,11 @@ func (in *serverInit) storeFirewall(ctx context.Context, members []pgha.Node, se
|
||||
} else {
|
||||
in.kept("firewall on %s", m.Name)
|
||||
}
|
||||
// And after, as the cluster's settings say, read from the
|
||||
// member's own copy of the store.
|
||||
if err := firewall.InstallFollower(ctx, m.Remote); err != nil {
|
||||
fmt.Fprintf(in.out, " !! firewall on %s: %v; the cluster's settings will not change it\n", m.Name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
@@ -135,5 +140,5 @@ func loopbackListen(listen string) bool {
|
||||
// ensureFirewallSync installs the units that apply the cluster's firewall
|
||||
// setting on this control plane.
|
||||
func (in *serverInit) ensureFirewallSync(ctx context.Context) error {
|
||||
return installFirewallSync(ctx, firewall.Local{})
|
||||
return installFirewallSync(ctx, firewall.Local{}, firewallStateFile)
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.yaml.in/yaml/v3"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
)
|
||||
|
||||
// Where the cluster says what went wrong: a model parked, a node lost, a
|
||||
// budget spent. notify.webhooks existed, and setup never asked for one, so a
|
||||
// site learnt of a lost node from its users.
|
||||
|
||||
// parseWebhook reads a webhook as setup takes it: an address, with "slack:"
|
||||
// or "json:" in front to say what it is sent, else what its address says.
|
||||
func parseWebhook(s string) (notify.Webhook, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
format := ""
|
||||
for _, f := range []string{"slack", "json"} {
|
||||
if rest, ok := strings.CutPrefix(s, f+":"); ok && !strings.HasPrefix(rest, "//") {
|
||||
format, s = f, rest
|
||||
}
|
||||
}
|
||||
u, err := url.Parse(s)
|
||||
if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" {
|
||||
return notify.Webhook{}, fmt.Errorf("webhook %q: an http(s) address, with slack: or json: in front to say its format", s)
|
||||
}
|
||||
if format == "" {
|
||||
format = webhookFormat(u)
|
||||
}
|
||||
return notify.Webhook{URL: s, Format: format}, nil
|
||||
}
|
||||
|
||||
// webhookFormat is what an address takes: Slack's own, and the incoming
|
||||
// webhooks that read Slack's {"text": …} — Mattermost's and Rocket.Chat's
|
||||
// /hooks/, Discord's …/slack — are sent that; anything else the event as
|
||||
// JSON.
|
||||
func webhookFormat(u *url.URL) string {
|
||||
switch {
|
||||
case u.Host == "hooks.slack.com",
|
||||
strings.Contains(u.Path, "/hooks/"),
|
||||
strings.HasSuffix(u.Path, "/slack"):
|
||||
return "slack"
|
||||
}
|
||||
return "json"
|
||||
}
|
||||
|
||||
// parseWebhooks reads a list of them, by commas or as given.
|
||||
func parseWebhooks(list []string) ([]notify.Webhook, error) {
|
||||
var out []notify.Webhook
|
||||
for _, item := range list {
|
||||
for _, s := range strings.Split(item, ",") {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
continue
|
||||
}
|
||||
w, err := parseWebhook(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, w)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// webhookEntry is one of notify.webhooks as the settings document has it.
|
||||
type webhookEntry struct {
|
||||
URL string `yaml:"url"`
|
||||
Format string `yaml:"format,omitempty"`
|
||||
Events []string `yaml:"events,omitempty,flow"`
|
||||
SecretFile string `yaml:"secret_file,omitempty"`
|
||||
}
|
||||
|
||||
// withWebhooks adds these webhooks to a settings document's notify.webhooks,
|
||||
// and says whether that changed it: one already there, by address, is left
|
||||
// as it is, with whatever events and secret it was given.
|
||||
func withWebhooks(doc *settings.Doc, add []notify.Webhook) (bool, error) {
|
||||
path := []string{"notify", "webhooks"}
|
||||
var list []webhookEntry
|
||||
if n := doc.Node(path); n != nil {
|
||||
if err := n.Decode(&list); err != nil {
|
||||
return false, fmt.Errorf("notify.webhooks in the cluster's settings: %w", err)
|
||||
}
|
||||
}
|
||||
changed := false
|
||||
for _, w := range add {
|
||||
if !slices.ContainsFunc(list, func(e webhookEntry) bool { return e.URL == w.URL }) {
|
||||
list = append(list, webhookEntry{URL: w.URL, Format: w.Format})
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
return false, nil
|
||||
}
|
||||
b, err := yaml.Marshal(list)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, doc.SetYAML(path, string(b))
|
||||
}
|
||||
|
||||
// ensureWebhooks puts the webhooks given into the cluster's settings and
|
||||
// sends each a test, once the control plane runs on them. Said, not fatal: a
|
||||
// cluster without its alerts still serves.
|
||||
func (in *serverInit) ensureWebhooks(ctx context.Context) error {
|
||||
if len(in.webhooks) == 0 {
|
||||
return nil
|
||||
}
|
||||
hooks, err := parseWebhooks(in.webhooks)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c := in.adminAPI()
|
||||
var cur adminapi.SettingsView
|
||||
if err := c.get(ctx, "/api/v1/settings", &cur); err != nil {
|
||||
return err
|
||||
}
|
||||
doc, err := settings.FromBytes("settings", []byte(cur.Body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
changed, err := withWebhooks(doc, hooks)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if changed {
|
||||
b, err := doc.Bytes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, _ := json.Marshal(adminapi.SettingsRequest{Base: cur.Version, Body: string(b), Note: "notification webhooks, from setup"})
|
||||
if err := c.put(ctx, "/api/v1/settings", req, &cur); err != nil {
|
||||
return err
|
||||
}
|
||||
in.done("notification webhooks in the cluster's settings (version %d)", cur.Version)
|
||||
} else {
|
||||
in.kept("notification webhooks (already in the cluster's settings)")
|
||||
}
|
||||
|
||||
// The control planes start again on the change; the test is sent by
|
||||
// one that knows every webhook asked for.
|
||||
var res []notify.Result
|
||||
err = waitFor(ctx, 2*time.Minute, "the control plane to run on its webhooks", func() (bool, error) {
|
||||
res = nil
|
||||
if c.post(ctx, "/api/v1/notify/test", nil, &res) != nil {
|
||||
return false, nil
|
||||
}
|
||||
for _, w := range hooks {
|
||||
// Answered by host: the path of a webhook is its secret.
|
||||
if !slices.ContainsFunc(res, func(r notify.Result) bool { return r.URL == notify.Redact(w.URL) }) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var failed []string
|
||||
for _, r := range res {
|
||||
if !slices.ContainsFunc(hooks, func(w notify.Webhook) bool { return notify.Redact(w.URL) == r.URL }) {
|
||||
continue
|
||||
}
|
||||
if r.Err != "" {
|
||||
failed = append(failed, r.URL+": "+r.Err)
|
||||
continue
|
||||
}
|
||||
in.done("a test notification reached %s", r.URL)
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return errors.New("a test notification did not arrive: " + strings.Join(failed, "; ") +
|
||||
" (kept in the settings: 'llmgw ctl notify test' tries again)")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
)
|
||||
|
||||
func TestAWebhooksFormatIsWhatItsAddressTakes(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"https://hooks.slack.com/services/T1/B2/abc": "slack",
|
||||
"https://chat.example.com/hooks/xyz": "slack", // Mattermost, Rocket.Chat
|
||||
"https://discord.com/api/webhooks/1/abc/slack": "slack",
|
||||
"https://alerts.example.com/llmgw": "json",
|
||||
"json:https://hooks.slack.com/services/T1/B2/abc": "json",
|
||||
"slack:https://alerts.example.com/llmgw": "slack",
|
||||
" https://events.pagerduty.com/integration/x/enqueue ": "json",
|
||||
} {
|
||||
w, err := parseWebhook(in)
|
||||
if err != nil || w.Format != want || strings.HasPrefix(w.URL, "slack:") || strings.HasPrefix(w.URL, "json:") {
|
||||
t.Errorf("%q: %+v %v, want %s", in, w, err, want)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"hooks.slack.com/x", "ftp://example.com/x", "slack:", "https://"} {
|
||||
if _, err := parseWebhook(bad); err == nil {
|
||||
t.Errorf("%q was taken", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebhooksAreAddedToTheSettingsBesideTheOnesThere(t *testing.T) {
|
||||
doc, err := settings.FromBytes("settings", []byte("router:\n retries: 2\nnotify:\n webhooks:\n"+
|
||||
" - url: https://alerts.example.com/a\n events: [node_lost]\n secret_file: /etc/llmgw/secrets/hook\n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hooks, _ := parseWebhooks([]string{"https://alerts.example.com/a, https://hooks.slack.com/services/T/B/c"})
|
||||
changed, err := withWebhooks(doc, hooks)
|
||||
if err != nil || !changed {
|
||||
t.Fatalf("changed %v: %v", changed, err)
|
||||
}
|
||||
b, _ := doc.Bytes()
|
||||
got := string(b)
|
||||
for _, want := range []string{"retries: 2", "url: https://alerts.example.com/a", "events: [node_lost]",
|
||||
"secret_file: /etc/llmgw/secrets/hook", "url: https://hooks.slack.com/services/T/B/c", "format: slack"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("%q not in:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
if strings.Count(got, "alerts.example.com/a") != 1 {
|
||||
t.Errorf("a webhook already there was added again:\n%s", got)
|
||||
}
|
||||
if changed, _ := withWebhooks(doc, hooks); changed {
|
||||
t.Error("adding the same webhooks again changed the settings")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetupPutsItsWebhooksInTheSettingsAndTestsThemOnceInForce(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
body, tests := "router:\n retries: 2\n", 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/settings":
|
||||
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 1, Body: body})
|
||||
case r.Method == http.MethodPut && r.URL.Path == "/api/v1/settings":
|
||||
var req adminapi.SettingsRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
body = req.Body
|
||||
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 2, Body: body})
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/api/v1/notify/test":
|
||||
tests++
|
||||
// The first answer comes from the control plane before it
|
||||
// started again on the change: it knows no webhooks.
|
||||
res := []notify.Result{}
|
||||
if tests > 1 {
|
||||
res = []notify.Result{{URL: notify.Redact("https://hooks.slack.com/services/T/B/c")},
|
||||
{URL: notify.Redact("https://ops.example.com/in"), Err: "502 Bad Gateway"}}
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(res)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "server.yaml"),
|
||||
[]byte("admin_listen: "+strings.TrimPrefix(srv.URL, "http://")+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out bytes.Buffer
|
||||
in := &serverInit{etcDir: dir, out: &out,
|
||||
webhooks: []string{"https://hooks.slack.com/services/T/B/c", "https://ops.example.com/in"}}
|
||||
err := in.ensureWebhooks(t.Context())
|
||||
if err == nil || !strings.Contains(err.Error(), "502") || !strings.Contains(err.Error(), "ctl notify test") {
|
||||
t.Errorf("a webhook that did not answer: %v", err)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !strings.Contains(body, "format: slack") || !strings.Contains(body, "url: https://ops.example.com/in") ||
|
||||
!strings.Contains(body, "retries: 2") || tests < 2 {
|
||||
t.Errorf("settings %q after %d tests", body, tests)
|
||||
}
|
||||
if strings.Contains(out.String(), "/services/T/B/c") {
|
||||
t.Errorf("a webhook's secret path was printed:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "reached https://hooks.slack.com/…") {
|
||||
t.Errorf("the one that answered is not said:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
@@ -348,15 +349,23 @@ func (in *serverInit) setupPostgres(ctx context.Context) (string, error) {
|
||||
}
|
||||
|
||||
if installed {
|
||||
// Its defaults assume it owns the machine. A gateway swapped out to
|
||||
// give the database cache is worse than a smaller cache, and a store
|
||||
// whose working set is megabytes needs little. Only a PostgreSQL init
|
||||
// installed itself: one that was already here may serve something
|
||||
// else that needs its connections.
|
||||
for _, q := range []string{
|
||||
"ALTER SYSTEM SET shared_buffers = '96MB'",
|
||||
"ALTER SYSTEM SET max_connections = 40",
|
||||
} {
|
||||
// Sized to the machine's memory, as one it shares with the gateway:
|
||||
// a gateway swapped out to give the database cache is worse than a
|
||||
// smaller cache, but a knowledge base's vector index is searched
|
||||
// well only from memory. Only a PostgreSQL init installed itself:
|
||||
// one that was already here may serve something else that needs its
|
||||
// connections.
|
||||
meminfo, _ := os.ReadFile("/proc/meminfo")
|
||||
total, ok := pgha.MemTotal(string(meminfo))
|
||||
if !ok {
|
||||
total = 1 << 30
|
||||
}
|
||||
mem := pgha.MemorySettings(total, true)
|
||||
qs := []string{"ALTER SYSTEM SET max_connections = 40"}
|
||||
for _, k := range []string{"shared_buffers", "effective_cache_size", "maintenance_work_mem", "work_mem"} {
|
||||
qs = append(qs, "ALTER SYSTEM SET "+k+" = '"+mem[k]+"'")
|
||||
}
|
||||
for _, q := range qs {
|
||||
if _, err := in.psql(ctx, q); err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -367,9 +376,12 @@ func (in *serverInit) setupPostgres(ctx context.Context) (string, error) {
|
||||
if err := in.waitForPostgres(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
in.done("PostgreSQL sized for a machine it shares (shared_buffers 96MB, 40 connections)")
|
||||
in.done("PostgreSQL sized for a machine it shares, with %d MB of memory (shared_buffers %s, work_mem %s, 40 connections)",
|
||||
total>>20, mem["shared_buffers"], mem["work_mem"])
|
||||
}
|
||||
|
||||
in.setupPgvector(ctx, discovery.DiscoverHost())
|
||||
|
||||
// Proof the address works, before anything is moved into it.
|
||||
address := "@" + in.dsnPath()
|
||||
s, err := store.Open(ctx, address)
|
||||
@@ -482,8 +494,17 @@ func lastLines(b []byte, n int) string {
|
||||
|
||||
// psql runs one statement as the postgres superuser, over the local socket.
|
||||
func (in *serverInit) psql(ctx context.Context, query string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, "runuser", "-u", "postgres", "--",
|
||||
in.pg.tool("psql"), "-v", "ON_ERROR_STOP=1", "-qAtX", "-c", query)
|
||||
return in.psqlIn(ctx, "", query)
|
||||
}
|
||||
|
||||
// psqlIn runs a statement as postgres in one database, or in postgres's
|
||||
// default one when db is "".
|
||||
func (in *serverInit) psqlIn(ctx context.Context, db, query string) (string, error) {
|
||||
args := []string{"-v", "ON_ERROR_STOP=1", "-qAtX", "-c", query}
|
||||
if db != "" {
|
||||
args = append(args, "-d", db)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "runuser", append([]string{"-u", "postgres", "--", in.pg.tool("psql")}, args...)...)
|
||||
// postgres cannot enter root's home, and psql says so on every call.
|
||||
cmd.Dir = "/"
|
||||
out, err := cmd.CombinedOutput()
|
||||
@@ -538,3 +559,50 @@ func (in *serverInit) stopService(ctx context.Context) (bool, error) {
|
||||
in.done("llmgw-server stopped while its store moves")
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// pgvectorPackages are the package names of pgvector for a PostgreSQL major
|
||||
// version: Debian's and Ubuntu's (and PGDG's apt repository) name it after the
|
||||
// server, PGDG's rpm repository after the version alone.
|
||||
func pgvectorPackages(major int) map[string][]string {
|
||||
return map[string][]string{
|
||||
"deb": {fmt.Sprintf("postgresql-%d-pgvector", major)},
|
||||
"rpm": {fmt.Sprintf("pgvector_%d", major)},
|
||||
}
|
||||
}
|
||||
|
||||
// setupPgvector installs pgvector and enables it in llmgw's database, for
|
||||
// knowledge bases. Not a reason to stop: without it everything else works,
|
||||
// and the store says why knowledge bases are off when somebody asks for one.
|
||||
func (in *serverInit) setupPgvector(ctx context.Context, h discovery.Host) {
|
||||
warn := func(format string, a ...any) {
|
||||
fmt.Fprintf(in.out, " !! pgvector: "+format+"; knowledge bases stay off\n", a...)
|
||||
}
|
||||
if on, err := in.psqlIn(ctx, pgDatabase, "SELECT 1 FROM pg_extension WHERE extname = 'vector'"); err == nil && on == "1" {
|
||||
in.kept("pgvector in database %s", pgDatabase)
|
||||
return
|
||||
}
|
||||
have, err := in.psqlBool(ctx, "SELECT 1 FROM pg_available_extensions WHERE name = 'vector'")
|
||||
if err != nil {
|
||||
warn("%v", err)
|
||||
return
|
||||
}
|
||||
if !have {
|
||||
num, err := in.psql(ctx, "SHOW server_version_num")
|
||||
n, convErr := strconv.Atoi(num)
|
||||
if err != nil || convErr != nil || h.Family() == "" {
|
||||
warn("cannot tell which package to install (version %q, %s)", num, h.OSID)
|
||||
return
|
||||
}
|
||||
pkgs := pgvectorPackages(n / 10000)
|
||||
pi := &provision.Installer{Host: h, Log: slog.Default()}
|
||||
if err := pi.InstallPackages(ctx, pkgs); err != nil {
|
||||
warn("installing %s: %v", strings.Join(pkgs[h.Family()], " "), err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if _, err := in.psqlIn(ctx, pgDatabase, "CREATE EXTENSION IF NOT EXISTS vector"); err != nil {
|
||||
warn("%v", err)
|
||||
return
|
||||
}
|
||||
in.done("pgvector enabled in database %s", pgDatabase)
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// llmgwServices are the units llmgw runs as on a machine, in the order they
|
||||
// start: the store's clients after the store, the server before the agent
|
||||
// that dials it.
|
||||
var llmgwServices = []string{"llmgw-mirror", "llmgw-server", "llmgw-agent"}
|
||||
|
||||
// unitDir is where llmgw's units are installed.
|
||||
var unitDir = "/etc/systemd/system"
|
||||
|
||||
// serviceRunner runs systemctl; a variable so tests stand in for it.
|
||||
var serviceRunner = func(ctx context.Context, args ...string) (string, error) {
|
||||
out, err := exec.CommandContext(ctx, "systemctl", args...).CombinedOutput() //nolint:gosec // fixed command
|
||||
return strings.TrimSpace(string(out)), err
|
||||
}
|
||||
|
||||
// installedServices are the llmgw units this machine has, or those named.
|
||||
func installedServices(ctx context.Context, names []string) ([]string, error) {
|
||||
var want []string
|
||||
for _, n := range names {
|
||||
if !strings.HasPrefix(n, "llmgw-") {
|
||||
n = "llmgw-" + n
|
||||
}
|
||||
want = append(want, n)
|
||||
}
|
||||
var have []string
|
||||
for _, u := range llmgwServices {
|
||||
if _, err := os.Stat(unitDir + "/" + u + ".service"); err != nil {
|
||||
continue
|
||||
}
|
||||
if len(want) > 0 && !hasUnit(want, u) {
|
||||
continue
|
||||
}
|
||||
have = append(have, u)
|
||||
}
|
||||
for _, w := range want {
|
||||
if !hasUnit(have, w) {
|
||||
return nil, fmt.Errorf("%s is not installed here (this machine has: %s)", w, strings.Join(present(), ", "))
|
||||
}
|
||||
}
|
||||
if len(have) == 0 {
|
||||
return nil, errors.New("no llmgw service is installed on this machine")
|
||||
}
|
||||
return have, nil
|
||||
}
|
||||
|
||||
func present() []string {
|
||||
var out []string
|
||||
for _, u := range llmgwServices {
|
||||
if _, err := os.Stat(unitDir + "/" + u + ".service"); err == nil {
|
||||
out = append(out, strings.TrimPrefix(u, "llmgw-"))
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return []string{"none"}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func hasUnit(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// newServiceCmd is llmgw service: this machine's llmgw services, one by one.
|
||||
// The whole system is llmgw status, start, stop and restart.
|
||||
func newServiceCmd() *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
Use: "service",
|
||||
Short: "This machine's llmgw services: status, start, stop, restart",
|
||||
Long: "The control plane (server), node agent (agent) and model mirror (mirror) on\n" +
|
||||
"this machine alone, by systemd. The whole system is 'llmgw status', 'llmgw\n" +
|
||||
"stop', 'llmgw start' and 'llmgw restart', run on a control plane.",
|
||||
}
|
||||
c.AddCommand(serviceVerbs()...)
|
||||
return c
|
||||
}
|
||||
|
||||
func serviceVerbs() []*cobra.Command {
|
||||
status := &cobra.Command{
|
||||
Use: "status [server|agent|mirror]...",
|
||||
Short: "Whether llmgw's services on this machine run",
|
||||
Long: "Each llmgw service this machine has — the control plane (server), the\n" +
|
||||
"node agent (agent), the model mirror (mirror) — and whether it runs.",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
units, err := localUnits(cmd, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
for _, u := range units {
|
||||
state, _ := serviceRunner(cmd.Context(), "is-active", u)
|
||||
since, _ := serviceRunner(cmd.Context(), "show", u, "-p", "ActiveEnterTimestamp", "--value")
|
||||
fmt.Fprintf(out, "%-14s %-10s %s\n", strings.TrimPrefix(u, "llmgw-"), state, since)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
verb := func(use, short, long, action string, reverse bool) *cobra.Command {
|
||||
var engines bool
|
||||
c := &cobra.Command{
|
||||
Use: use + " [server|agent|mirror]...",
|
||||
Short: short,
|
||||
Long: long,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if os.Geteuid() != 0 {
|
||||
return errors.New("starting and stopping services needs root: run it with sudo")
|
||||
}
|
||||
units, err := localUnits(cmd, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if reverse {
|
||||
// The agent before the server it dials, the server before
|
||||
// the mirror it reads.
|
||||
for i, j := 0, len(units)-1; i < j; i, j = i+1, j-1 {
|
||||
units[i], units[j] = units[j], units[i]
|
||||
}
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
for _, u := range units {
|
||||
if msg, err := serviceRunner(cmd.Context(), action, u); err != nil {
|
||||
return fmt.Errorf("%s %s: %v %s", action, u, err, msg)
|
||||
}
|
||||
state, _ := serviceRunner(cmd.Context(), "is-active", u)
|
||||
fmt.Fprintf(out, "%-14s %s\n", strings.TrimPrefix(u, "llmgw-"), state)
|
||||
}
|
||||
if action == "stop" && hasUnit(units, "llmgw-agent") {
|
||||
if engines {
|
||||
_, _ = exec.CommandContext(cmd.Context(), "pkill", "-u", "llmgw-engine").CombinedOutput() //nolint:gosec // fixed command
|
||||
fmt.Fprintln(out, "engines stopped (every process of llmgw-engine)")
|
||||
} else {
|
||||
fmt.Fprintln(out, "\nThe engines keep serving: they outlive the agent, which takes them over again when it starts.\n"+
|
||||
"--engines stops them too; the control plane then places their replicas elsewhere.")
|
||||
}
|
||||
}
|
||||
if action == "stop" && hasUnit(units, "llmgw-server") {
|
||||
fmt.Fprintln(out, "A standby control plane, where there is one, takes over within seconds.")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
if action == "stop" {
|
||||
c.Flags().BoolVar(&engines, "engines", false, "also stop the engines this node runs")
|
||||
}
|
||||
return c
|
||||
}
|
||||
return []*cobra.Command{
|
||||
status,
|
||||
verb("start", "Start llmgw's services on this machine",
|
||||
"Starts each llmgw service this machine has, or those named: the mirror, the\n"+
|
||||
"control plane, the node agent, in that order.", "start", false),
|
||||
verb("stop", "Stop llmgw's services on this machine",
|
||||
"Stops each llmgw service this machine has, or those named: the agent, the\n"+
|
||||
"control plane, the mirror, in that order. The engines a node runs keep\n"+
|
||||
"serving unless --engines is given; a standby control plane takes over from\n"+
|
||||
"a leader stopped. They stay stopped until 'llmgw start', or the machine\n"+
|
||||
"boots.", "stop", true),
|
||||
verb("restart", "Restart llmgw's services on this machine",
|
||||
"Restarts each llmgw service this machine has, or those named. Engines keep\n"+
|
||||
"serving and are taken over again by the agent; a control plane's settings\n"+
|
||||
"and server.yaml are read anew.", "restart", false),
|
||||
}
|
||||
}
|
||||
|
||||
func localUnits(cmd *cobra.Command, args []string) ([]string, error) {
|
||||
if runtime.GOOS != "linux" {
|
||||
return nil, fmt.Errorf("llmgw's services run under systemd on Linux; this is %s", runtime.GOOS)
|
||||
}
|
||||
return installedServices(cmd.Context(), args)
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
func TestTheServicesHereAreFoundAndAskedAbout(t *testing.T) {
|
||||
old, oldRun := unitDir, serviceRunner
|
||||
unitDir = t.TempDir()
|
||||
t.Cleanup(func() { unitDir, serviceRunner = old, oldRun })
|
||||
for _, u := range []string{"llmgw-server", "llmgw-agent"} {
|
||||
_ = os.WriteFile(filepath.Join(unitDir, u+".service"), []byte("[Unit]"), 0o644)
|
||||
}
|
||||
var asked []string
|
||||
serviceRunner = func(_ context.Context, args ...string) (string, error) {
|
||||
asked = append(asked, strings.Join(args, " "))
|
||||
if args[0] == "is-active" {
|
||||
return "active", nil
|
||||
}
|
||||
return "Wed 2026-10-07 12:00:00 UTC", nil
|
||||
}
|
||||
|
||||
units, err := installedServices(context.Background(), nil)
|
||||
if err != nil || strings.Join(units, ",") != "llmgw-server,llmgw-agent" {
|
||||
t.Fatalf("%v %v", units, err)
|
||||
}
|
||||
if _, err := installedServices(context.Background(), []string{"mirror"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "server, agent") {
|
||||
t.Errorf("a service not here: %v", err)
|
||||
}
|
||||
if u, err := installedServices(context.Background(), []string{"agent"}); err != nil || len(u) != 1 {
|
||||
t.Errorf("one named: %v %v", u, err)
|
||||
}
|
||||
|
||||
if runtime.GOOS != "linux" {
|
||||
return
|
||||
}
|
||||
var out bytes.Buffer
|
||||
c := serviceVerbs()[0]
|
||||
c.SetOut(&out)
|
||||
c.SetContext(context.Background())
|
||||
if err := c.RunE(c, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "server") || !strings.Contains(out.String(), "active") {
|
||||
t.Errorf("%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopSetsTheClustersServingStopped(t *testing.T) {
|
||||
var put adminapi.SettingsRequest
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/api/v1/summary":
|
||||
_ = json.NewEncoder(w).Encode(adminapi.Summary{})
|
||||
case r.Method == "GET" && r.URL.Path == "/api/v1/settings":
|
||||
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 3, Body: "router:\n retries: 2\n"})
|
||||
case r.Method == "PUT" && r.URL.Path == "/api/v1/settings":
|
||||
_ = json.NewDecoder(r.Body).Decode(&put)
|
||||
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 4, Body: put.Body})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
dir := t.TempDir()
|
||||
cfg := filepath.Join(dir, "server.yaml")
|
||||
_ = os.WriteFile(filepath.Join(dir, "token"), []byte("t"), 0o600)
|
||||
_ = os.WriteFile(cfg, []byte("listen: \":8080\"\nagent_insecure: true\nadmin_listen: "+
|
||||
strings.TrimPrefix(srv.URL, "http://")+"\nadmin_remote: true\nadmin_token_file: "+filepath.Join(dir, "token")+"\n"), 0o600)
|
||||
|
||||
var stop *cobra.Command
|
||||
for _, c := range newSystemCmds() {
|
||||
if c.Name() == "stop" {
|
||||
stop = c
|
||||
}
|
||||
}
|
||||
var out bytes.Buffer
|
||||
stop.SetOut(&out)
|
||||
stop.SetArgs([]string{"--config", cfg, "--note", "maintenance"})
|
||||
if err := stop.ExecuteContext(context.Background()); err != nil {
|
||||
t.Fatalf("%v: %s", err, out.String())
|
||||
}
|
||||
if put.Base != 3 || !strings.Contains(put.Body, "stopped: true") || !strings.Contains(put.Body, "retries: 2") ||
|
||||
put.Note != "maintenance" {
|
||||
t.Errorf("sent %+v", put)
|
||||
}
|
||||
if !strings.Contains(out.String(), "stopping") {
|
||||
t.Errorf("%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Started while the control plane has not yet taken the setting up: every
|
||||
// model of a stopped system has the nothing it asks for, which is not
|
||||
// serving.
|
||||
func TestStartWaitsForTheControlPlaneToServe(t *testing.T) {
|
||||
old := servicePoll
|
||||
servicePoll = time.Millisecond
|
||||
t.Cleanup(func() { servicePoll = old })
|
||||
answers := []adminapi.Summary{
|
||||
{Stopped: true, Models: adminapi.ModelTotals{Total: 2, Satisfied: 2}},
|
||||
{Models: adminapi.ModelTotals{Total: 2, Satisfied: 0}},
|
||||
{Models: adminapi.ModelTotals{Total: 2, Satisfied: 2}},
|
||||
}
|
||||
asked := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(answers[min(asked, len(answers)-1)])
|
||||
asked++
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
var out bytes.Buffer
|
||||
cmd := &cobra.Command{}
|
||||
cmd.SetOut(&out)
|
||||
cmd.SetContext(context.Background())
|
||||
c := &ctlClient{base: srv.URL, http: srv.Client()}
|
||||
if err := waitForService(cmd, func() *ctlClient { return c }, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked != 3 {
|
||||
t.Errorf("returned after %d answers: %s", asked, out.String())
|
||||
}
|
||||
}
|
||||
Loaded 100 of 415 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user