Compare commits
No files matched your search
@@ -39,6 +39,14 @@ jobs:
|
||||
- name: Build
|
||||
run: go build ./...
|
||||
|
||||
# llmgw-kb runs on people's own machines, Windows included, which no
|
||||
# test here runs on: building it for each catches a Unix-only call.
|
||||
- name: Build llmgw-kb for every system
|
||||
run: |
|
||||
for target in windows/amd64 windows/arm64 darwin/amd64 darwin/arm64 linux/amd64 linux/arm64; do
|
||||
CGO_ENABLED=0 GOOS="${target%/*}" GOARCH="${target#*/}" go build -o /dev/null ./cmd/llmgw-kb
|
||||
done
|
||||
|
||||
# The Terraform provider is a module of its own, which ./... above does
|
||||
# not reach. Its tests check its client against docs/admin-api.openapi.json,
|
||||
# so a change to the admin API that breaks the provider fails here. The
|
||||
|
||||
@@ -55,6 +55,21 @@ jobs:
|
||||
(cd dist && sha256sum "${name}" >> checksums.txt)
|
||||
done
|
||||
|
||||
# llmgw-kb, which people run on their own machines to put their
|
||||
# documents into a knowledge base: Windows too, which llmgw itself
|
||||
# does not build for.
|
||||
for target in windows/amd64 windows/arm64 darwin/amd64 darwin/arm64 linux/amd64 linux/arm64; do
|
||||
os="${target%/*}"; arch="${target#*/}"
|
||||
ext=""; [ "$os" = windows ] && ext=".exe"
|
||||
name="llmgw-kb-${version}-${os}-${arch}${ext}"
|
||||
echo "building $name"
|
||||
CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build \
|
||||
-trimpath \
|
||||
-ldflags "-s -w -X gitea.mynodes.xyz/ilker/llmgw/internal/version.Version=${version}" \
|
||||
-o "dist/${name}" ./cmd/llmgw-kb
|
||||
(cd dist && sha256sum "${name}" >> checksums.txt)
|
||||
done
|
||||
|
||||
# The reverse proxy configurations, so a release page carries
|
||||
# everything install.md asks for and the repository is not needed.
|
||||
for f in deploy/proxy/Caddyfile deploy/proxy/nginx.conf; do
|
||||
@@ -91,6 +106,7 @@ jobs:
|
||||
# test the cross-compiled ones, which is why the native build is
|
||||
# checked rather than assumed.
|
||||
./dist/llmgw-${version}-linux-amd64 --version
|
||||
./dist/llmgw-kb-${version}-linux-amd64 --version
|
||||
cat dist/checksums.txt
|
||||
|
||||
# Signed, so a machine that trusts the key built into it installs
|
||||
|
||||
+178
@@ -818,6 +818,184 @@ The same document, edited on /admin/settings, for whoever has no terminal.
|
||||
|
||||
---
|
||||
|
||||
## F. Company documents: knowledge bases (RAG), then adapters (2026-10-08)
|
||||
|
||||
A company wants a model to answer from its own documents. Retrieval is the
|
||||
tool for *knowledge* (sources shown, re-indexed in minutes, access per
|
||||
tenant); LoRA training is for *behaviour* (vocabulary, format) and has no
|
||||
access control — whatever goes into weights reaches everyone who may use the
|
||||
adapter. RAG first. Decisions taken: access per collection (no per-document
|
||||
ACL yet); a personal key or session names a project or team and gets that
|
||||
chain only, never the union of its person's memberships; PostgreSQL store
|
||||
only, off on SQLite with a sentence saying why.
|
||||
|
||||
### F1. Store: collections, grants, chunks (M) — done 2026-10-08 (branch `rag`)
|
||||
Built (`internal/store/knowledge.go`, migration 0035 PostgreSQL only):
|
||||
- `kb_collections` (owner scope/id, org, embed_model, dims), `kb_grants`
|
||||
(read|write, any scope, never across organizations; a key needs a
|
||||
project; a person needs a membership in the organization),
|
||||
`kb_documents` (digest: the same again is no change, a new one replaces
|
||||
in one transaction).
|
||||
- `kb_chunks` is created when the store opens, not in a migration (a
|
||||
migration needing an extension would stop the control plane; one that
|
||||
skipped it would never run again): partitioned by organization (hashed
|
||||
partition name), `tsv` generated (`simple`), GIN on the parent, one
|
||||
partial HNSW index per collection on its organization's partition
|
||||
(`embedding::vector(dims)`, `WHERE collection = '<id>'`). EXPLAIN on
|
||||
20k rows: partition pruned, that index used, policy applied.
|
||||
- RLS FORCEd on `kb_chunks` against `llmgw.kb_org`/`llmgw.kb_collections`
|
||||
(set with set_config per transaction), one role — the table owner — not
|
||||
a separate one. Chunks are deleted by the foreign-key cascade from their
|
||||
document, which RLS does not block.
|
||||
- `CollectionsFor(Effective, write)`: owner or grant anywhere on the chain;
|
||||
no organization (personal key without a project) reads nothing.
|
||||
- `SearchChunks`: per collection nearest-neighbour (literal id and cast so
|
||||
the partial index is used) plus a full-text arm with any word
|
||||
(`a | b`, lowercased by the database, not Go), fused by reciprocal rank.
|
||||
- Deleting a project/team/org that owns collections is refused like one
|
||||
with members (changed from "deleted with them": every other delete in the
|
||||
tree refuses); DeleteUser/ForgetUser delete the person's collections;
|
||||
deleting an organization drops its partition; grants to a deleted scope go.
|
||||
- `server init` installs `postgresql-N-pgvector` / `pgvector_N` and enables
|
||||
it; pgha installs it on every member at setup and enables it on the
|
||||
leader on every run. Neither stops on failure.
|
||||
Left: a cluster set up before (the PoC) gets no package from a rerun of
|
||||
pgha on a running cluster — install by hand or add an upgrade step;
|
||||
`server restore`/`CopyTo` do not carry kb tables (SQLite → PostgreSQL only,
|
||||
and SQLite has none); tests need `LLMGW_TEST_POSTGRES` pointing at a
|
||||
pgvector database as a non-superuser (a superuser bypasses RLS).
|
||||
|
||||
### F2. Ingest on the user's machine (L) — done 2026-10-08 (branch `rag`)
|
||||
Uploading whole files to the control plane was dropped: a PDF is 20–100×
|
||||
its text, and the work belongs where the files are. Instead a small client,
|
||||
`llmgw-kb`, run by the person on their own machine with their own key:
|
||||
- **Its own binary** (`cmd/llmgw-kb`), for Windows, macOS and Linux on amd64
|
||||
and arm64. `llmgw` itself does not build on Windows (agent syscalls), and
|
||||
employees' laptops should not carry the server, agent and setup screens. A
|
||||
test keeps server packages out of its imports.
|
||||
- `llmgw-kb sync <collection> <dir|file…> [--include/--exclude globs]
|
||||
[--dry-run] [--delete]`: scan → local catalog (path, size, mtime, sha256;
|
||||
cache under os.UserCacheDir, digests not recomputed for unchanged files)
|
||||
→ the collection's manifest from the gateway (name → digest only) → diff
|
||||
→ for new/changed files only: extract, chunk, embed through the gateway's
|
||||
`/v1/embeddings` with the person's key (its accounting, limits, guard and
|
||||
band apply), and send name, digest, chunk texts and vectors; `--delete`
|
||||
removes documents whose files are gone. `--dry-run` lists what would leave
|
||||
the machine and how much. A file that cannot be read (locked, unsupported,
|
||||
no text) is skipped with the reason; the run goes on.
|
||||
- **Formats, as many as possible.** Built in, pure Go: PDF (text layer),
|
||||
DOCX/XLSX/PPTX, ODT/ODS/ODP, RTF, EPUB, HTML/XHTML, EML (text and HTML
|
||||
parts), Jupyter notebooks, subtitles (SRT/VTT), CSV/TSV, JSON/YAML/XML,
|
||||
Markdown/RST/AsciiDoc/Org, plain text and source code; ZIP archives
|
||||
recursed into. With a helper found on the machine: legacy DOC/XLS/PPT, RTF
|
||||
and anything else LibreOffice opens (`soffice --headless --convert-to`),
|
||||
images and scanned PDFs through Tesseract (`tur+eng`). Text in UTF-8,
|
||||
UTF-16 (BOM) or Windows-1254/1252 is converted to UTF-8.
|
||||
- **Across systems:** document names relative to the synced root, `/`
|
||||
separated, Unicode NFC (macOS hands out NFD), a warning for two names that
|
||||
differ only in case; CRLF → LF; key from `LLMGW_API_KEY` or `llmgw-kb
|
||||
login` (os.UserConfigDir, 0600 where that means something); the system's
|
||||
CA store, `--ca` for another.
|
||||
- **Gateway endpoints** (client listener, person's key):
|
||||
`GET /v1/kb/collections` (what this key may read/write),
|
||||
`GET /v1/kb/collections/{id}/manifest`, `PUT
|
||||
/v1/kb/collections/{id}/documents` (one document per request: name,
|
||||
digest, bytes, chunks with text and embedding; dims checked, chunk count
|
||||
and size bounded), `DELETE …/documents?name=`. Write needs a write grant
|
||||
or ownership on the key's chain (`CollectionsFor(…, write)`); otherwise
|
||||
404.
|
||||
- **Administration** stays with operators: `llmgw ctl kb
|
||||
create|list|delete|grant|revoke|docs`, admin API `/api/v1/kb`, audit,
|
||||
console screen; `docs/knowledge.md`.
|
||||
- Trade-off accepted: vectors come from the client, so a writer could store
|
||||
vectors that do not match their text — only in collections they may
|
||||
already write.
|
||||
Built: `internal/kb/extract` (formats above; helpers found on PATH and in
|
||||
each system's install places), `internal/kb/chunk` (≈1800 characters under
|
||||
the heading path, a paragraph of overlap, sections kept apart once a chunk
|
||||
is a third full), `internal/kb/kbsync` (ledger in SQLite at
|
||||
os.UserConfigDir/llmgw-kb/ledger.db: documents keyed by gateway, collection
|
||||
and name with file, size, mtime, digest, state, reason; runs), `cmd/llmgw-kb`
|
||||
(login, collections, sync, status, history, formats), `internal/router/kb.go`
|
||||
(the four gateway endpoints, `X-Llmgw-Project` for personal keys, audit),
|
||||
`internal/control/adminapi/kb.go` + `llmgw ctl kb` + console Knowledge
|
||||
screen, `docs/knowledge.md`; CI builds the client for all six targets and
|
||||
releases attach it. Tested: every format from generated fixtures and a
|
||||
Turkish PDF (OCR path with Tesseract here), sync against a fake gateway and
|
||||
end to end against the router on pgvector.
|
||||
Tried on the PoC 2026-10-08 (control planes 140 and 150 on a dev build of
|
||||
this branch; their previous binary is /usr/local/bin/llmgw.pre-rag): Ubuntu's
|
||||
pgvector 0.6.0 installed on 149/150 and enabled; bge-m3's weights were never
|
||||
in the mirror (fetched from CompendiumLabs/bge-m3-gguf@1864f47f) and it is
|
||||
held at 1 replica; tree Demo/ik/belgeler and Demo/satis/crm with a key each;
|
||||
collection Demo/politikalar filled from a Mac through an SSH tunnel with ten
|
||||
Turkish files in nine formats (9 sent in 5 s, the .doc skipped for want of
|
||||
LibreOffice; a rerun sent nothing in 0 s; a changed file alone was sent).
|
||||
Nine Turkish questions each found their page first once search scored by
|
||||
similarity plus word coverage (rank fusion had got two wrong). The other
|
||||
team's key saw no collection and got 404 by id; with a read grant it read
|
||||
and could not write; revoked, nothing again. On the live store the llmgw
|
||||
role (not superuser, no BYPASSRLS) sees 0 chunks with no scope set and 0
|
||||
with the organization alone.
|
||||
Left: the gateway's collection list shows the owner as team:ID, not its
|
||||
path; no LibreOffice on the dev machine, so the legacy
|
||||
Office path is untested beyond its error; the client is unsigned (Gatekeeper
|
||||
and SmartScreen warn); Outlook `.msg` is not read; one document over ~60 MiB
|
||||
of JSON (a few thousand pages) is refused rather than split.
|
||||
PostgreSQL tests of store, router and adminapi share LLMGW_TEST_POSTGRES and
|
||||
the store's TRUNCATE each test; run them with `go test -p 1` or one package
|
||||
at a time, or a router test loses its rows mid-run.
|
||||
|
||||
### F3. Retrieval on the request path (M) — done 2026-10-08 (branch `rag`)
|
||||
- `llmgw.knowledge: [collections]` in the body (stripped before the
|
||||
engine), or default collections per project/key.
|
||||
- Allowed set from the caller's chain (§7.9.2), cached like limits; grants
|
||||
add up down the tree. Personal keys and sessions need `x-llmgw-project`
|
||||
(or the playground's picker) naming a scope the person is a member of.
|
||||
- A collection not allowed answers 404, exactly as a missing one.
|
||||
- Hybrid search with the filter inside the query (per collection, its own
|
||||
partial index; pgvector 0.6 is enough —
|
||||
scored by similarity plus the share of the question's words held (built
|
||||
in F1/F2 as SearchChunks), `/v1/rerank` when a reranker is in the catalog. Top
|
||||
chunks into a system message with document and position;
|
||||
`x-llmgw-sources` lists them.
|
||||
- Leaks closed: answer-cache key includes the allowed set; per-organization
|
||||
`cache_salt` on RAG requests; injected passages under `no_store` and the
|
||||
guard's prompt-injection check; returned doc ids in the audit.
|
||||
- Done: router tests that tenant B never receives tenant A's chunks
|
||||
(including via cache), that a personal key without a project is refused,
|
||||
metrics (retrieval latency, chunks returned), docs.
|
||||
|
||||
Built (`internal/router/rag.go`): `X-Llmgw-Knowledge` (header, as the
|
||||
other llmgw extensions are, not a body field), `X-Llmgw-Knowledge-K`;
|
||||
collections from the key's chain (kbEffective, shared with the kb
|
||||
endpoints), 404 for one not readable; the question embedded through the
|
||||
ordinary path on the caller's principal; SearchChunks; passages trimmed to
|
||||
the model's context; added to the one leading system message (or a new
|
||||
one); `x-llmgw-sources`; before guardInput; never cached; metrics. Default collections built (kb_defaults on org/team/project/key, narrowest
|
||||
wins, 30 s cache, `ctl kb default`, console action; never refuse); the
|
||||
portal chat's Documents picker (collection, default or none; sources listed
|
||||
under the answer; a header the page sends itself is dropped). `cache_salt` per organization on requests given passages, sent to vLLM
|
||||
replicas only (chosen replica's engine). Checked on the PoC 2026-10-08:
|
||||
vLLM 0.30.0 (cuda01) with TinyLlama-1.1B AWQ run by hand beside the 35B in
|
||||
the card's last 2.3 GB — a 668-token prompt reused 656 tokens under the same
|
||||
salt and 0 under another's, with identical text (llmgw cannot place it
|
||||
there: placement takes whole GPUs). Live on the PoC: a team default answers
|
||||
a request without the header, `none` turns it off, an explicit header with
|
||||
k=2 gives two sources and no more, llmgw-kb lists owners by path. Left: returned passages
|
||||
are not audited per request (the header and the kept request record them).
|
||||
|
||||
### F4. Adapter training (L, after F1–F3)
|
||||
A run-to-completion job placed by the control plane like a replica and
|
||||
supervised by the agent like an engine, in a Python environment the agent
|
||||
installs (TRL + PEFT, QLoRA). Data: Q/A pairs generated from a collection by
|
||||
a cluster model through `/v1/batches`, reviewed by people. Output to the
|
||||
mirror, into the model's `adapters`, through canary and a quality suite.
|
||||
Runs when the scaling schedule frees a GPU; 8–14B dense bases on one card.
|
||||
Never train on a collection some users of the adapter may not read.
|
||||
|
||||
---
|
||||
|
||||
## D. Facts worth knowing before starting
|
||||
|
||||
- Admin API: `127.0.0.1:9090`, no auth, `ctlClient` in `internal/cli/ctl.go`.
|
||||
|
||||
+216
@@ -17,11 +17,227 @@ is only released once its **exit criterion** is met, not when its code lands.
|
||||
| v0.10.0 | — | InfiniBand fabrics and NVLink, doctor, node add, engines installed by the nodes | released |
|
||||
| v0.11.0 | — | High availability from setup, the setup and console screens, usage sums and per-tenant alerts | released |
|
||||
| v0.12.0 | — | The cluster's settings in the store, in force when changed; the firewall switch; wipe | released |
|
||||
| v0.13.0 | — | The whole system stopped and started from one command; Qwen3Guard judges, on a GPU and CPUs together | released |
|
||||
| v0.14.0 | — | Knowledge bases: documents from people's own machines, kept apart per tenant, answered from | released |
|
||||
| v1.0.0 | 5 | Tenancy, prefix affinity, benchmarking | planned |
|
||||
|
||||
Until v1.0.0 the minor version is the breaking one: configuration formats and Go
|
||||
APIs change between minors without a deprecation period.
|
||||
|
||||
## v0.14.0 — 2026-10-08
|
||||
|
||||
Knowledge bases: a company's documents, put in by people from their own
|
||||
machines, kept apart per tenant, and answered from.
|
||||
|
||||
**Upgrading.**
|
||||
- **pgvector on a cluster set up before this release.** Knowledge bases
|
||||
need pgvector, which `server init` and the high-availability database
|
||||
setup now install, but a rerun on a running cluster does not:
|
||||
1. Install `postgresql-<version>-pgvector` on every database machine.
|
||||
2. As postgres, run `CREATE EXTENSION vector` in llmgw's database.
|
||||
3. Restart the control planes.
|
||||
Until then everything else works, and `llmgw ctl kb list` says why
|
||||
knowledge bases are off. On a SQLite store they stay off.
|
||||
- **An embedding model.** A collection needs a catalog model with
|
||||
`capabilities.embeddings`, for example bge-m3, whose weights are in the
|
||||
mirror.
|
||||
- **The client.** People download `llmgw-kb` for their system from this
|
||||
release.
|
||||
|
||||
### Added
|
||||
|
||||
- **Collections and who may read them.** A collection is owned by an
|
||||
organization, team, project or person. Grants add up down the tenancy
|
||||
tree and never cross an organization. A collection a key may not read is
|
||||
answered as one that does not exist. Chunks are kept in PostgreSQL with
|
||||
pgvector, partitioned by organization, with an HNSW index per collection,
|
||||
and row-level security on the table as a second line behind every
|
||||
query's own filter. A project, team or organization that owns collections
|
||||
is not deleted; a person's collections go with their account.
|
||||
- **`llmgw-kb`**, its own download for Windows, macOS and Linux, run by
|
||||
people with their own key. It reads documents where they are and sends
|
||||
text and vectors, never the files, for what changed only; a SQLite ledger
|
||||
on the machine remembers what it did.
|
||||
- It reads PDF, Word, Excel, PowerPoint, OpenDocument, RTF, EPUB, HTML,
|
||||
MHTML, mail with attachments, notebooks, subtitles, ZIP archives, and
|
||||
text and source code in any common encoding.
|
||||
- Legacy Office files are read with LibreOffice, and scanned pages with
|
||||
Tesseract, when those are installed.
|
||||
- `--dry-run` says what would leave the machine; `--delete` removes only
|
||||
what that machine sent.
|
||||
- **Answers from documents.** A chat request names collections in
|
||||
`X-Llmgw-Knowledge`, or its part of the tree has a default (`llmgw ctl kb
|
||||
default`). The best passages, chosen by meaning plus a little for the
|
||||
question's words, are added to the system message, numbered, and
|
||||
`x-llmgw-sources` says which were used. This happens before the
|
||||
guardrails judge the request, and the answer is never served from the
|
||||
answer cache. On vLLM replicas the organization's `cache_salt` keeps
|
||||
tenants' prefix caches apart. It works through the Responses and Messages
|
||||
APIs too.
|
||||
- **Administration**: `llmgw ctl kb` (create, list, grant, revoke, default,
|
||||
docs, rm, search, delete), `/api/v1/kb`, the console's Knowledge screen,
|
||||
and a Documents picker on the portal's chat page.
|
||||
- Metrics `llmgw_kb_retrievals_total` and `llmgw_kb_retrieval_seconds`;
|
||||
[docs/knowledge.md](docs/knowledge.md).
|
||||
|
||||
## v0.13.0 — 2026-10-08
|
||||
|
||||
The whole system stopped, started and restarted from one command, a
|
||||
guardrails judge that no longer refuses ordinary coding requests, and the
|
||||
rest of the field report from v0.10.6.
|
||||
|
||||
**Upgrading.** A cluster keeps the judge its policies name: one installed
|
||||
before this release goes on asking Llama Guard 3 1B until `llmgw ctl guard
|
||||
judge use qwen3guard-0.6b`, which the leader then fetches and serves by
|
||||
itself, and which replaces the policies' Llama Guard category codes.
|
||||
|
||||
### Added
|
||||
|
||||
- **A judge on a GPU and one on CPUs together**: a policy's `judge.models`
|
||||
is a list asked in turn, the first that answers giving the verdict; the
|
||||
next is asked when one has no replica serving, fails, or misses its
|
||||
`latency_budget`. `llmgw ctl guard judge use qwen3guard-4b
|
||||
qwen3guard-0.6b` sets the order for every policy with a judge, and
|
||||
`guard judge status` says which judge stands in for which. A judge on a
|
||||
GPU is served as a `standard` model, so it takes a card only from what
|
||||
may give one up.
|
||||
- **A judge a policy names is fetched if it is missing**: the leader
|
||||
fetches the weights of a judge llmgw knows (`qwen3guard-0.6b`, `-4b`,
|
||||
`-8b`, `llama-guard-3-1b`, `-8b`) into the mirror and serves it, when a
|
||||
policy names it and the catalog does not have it — and again if it is
|
||||
removed. `guard judge enable --gpu` serves Qwen3Guard-Gen 4B on the class
|
||||
with the most free cards.
|
||||
- **The whole system stops, starts and restarts from one command**: `llmgw
|
||||
stop` takes every model down and answers every request for one with 503
|
||||
(`system_stopped`), keeping the control planes, agents and mirror running
|
||||
and no model's definition changed; `llmgw start` brings every model back.
|
||||
It is the cluster's setting `serving.stopped`, live on every control plane.
|
||||
`llmgw restart` restarts every node's agent and every control plane, the
|
||||
leader last, with the engines serving throughout. `llmgw status` says
|
||||
whether the system serves. One machine's services are `llmgw service
|
||||
status|start|stop|restart`.
|
||||
- **A replica says what it computes on**: `ctl replicas` (ON), the console
|
||||
and the admin API show `CUDA` with the GPUs its processes hold memory on
|
||||
and how much, as nvidia-smi sees them, or `CPU` — for every engine, kept by
|
||||
the agent across its own and the control plane's restarts. llama.cpp's
|
||||
layer count is added when it says it, which it does only at `-lv 4`.
|
||||
- **`ctl engine-check llamacpp` computes a token on the GPU**: a small GGUF
|
||||
(`--model`, or the smallest in the weights cache) is loaded with every layer
|
||||
on the GPU at `-lv 4` and asked for one token; a build whose CUDA backend
|
||||
does not load fails, by llama.cpp's own account of its layers. With no
|
||||
`--bin` it checks the build the agent installed.
|
||||
- **Engine versions and compute capability on nodes**: `ctl nodes` shows
|
||||
`llamacpp b10991` and `6000-24g (7.5)`; the admin API gives
|
||||
`engine_versions` and `compute_capabilities`. A profile's `engine_version`
|
||||
restricts it to nodes whose engine version starts with it, and
|
||||
explain-placement says which version a node has, or which engines.
|
||||
- **`ctl restart --max-surge N` and `--max-unavailable N`**: that many of a
|
||||
model's replicas restart at once, beside themselves or stopped first.
|
||||
- **Multi-node readiness apart from the node's**: doctor gives both, with
|
||||
fabric findings in a section of their own; `node add` says the multi-node
|
||||
one; `ctl nodes` shows a declared fabric's adapters that are down.
|
||||
- **A forgotten password is set through a link**: `/forgot` mails one when
|
||||
the cluster's `mail` settings say how (SMTP, sender, the public
|
||||
`base_url` it points at), saying the same whether or not the address has an
|
||||
account; `/reset` sets the password once and closes every session. Without
|
||||
mail, `llmgw users reset-link` makes the link for an operator to hand over.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **A replica restarted in place keeps its room**: the replica was stopped
|
||||
and the pass went on to other models before placing its replacement, so a
|
||||
model waiting for room took the freed cards. On the reference cluster a
|
||||
GPU judge waiting for the only card took it from the model being
|
||||
restarted there. The replacement is now placed in the same step.
|
||||
- **The judge no longer refuses ordinary coding requests**: Llama Guard 3
|
||||
1B, the judge `server init` installed, could not tell "continue", "devam
|
||||
et" or "how do I kill all child processes in bash" (0.51 to 0.92 unsafe)
|
||||
from harmful requests (0.97 to 1.00), at any quantisation. The judge is
|
||||
now Qwen3Guard-Gen 0.6B (format `qwen3guard`), which gave the same
|
||||
ordinary requests 0.15 at the most and the harmful ones 1.00, on CPUs,
|
||||
in 0.8 GB. Its third verdict, controversial, counts as safe unless the
|
||||
policy says `controversial: unsafe`. `guard judge use qwen3guard-0.6b`
|
||||
moves an installed cluster to it, and replaces Llama Guard's category
|
||||
codes, which Qwen3Guard never names, with the baseline's.
|
||||
- **A long message is judged rather than refused**: a judge read the whole
|
||||
last user message, and one with a file pasted into it overflowed the
|
||||
judge's context, so it gave no verdict and `fail: closed` refused the
|
||||
request. It now reads the start and the end, about 6000 characters.
|
||||
- **A replica no longer stays "starting" while it serves after a restart**:
|
||||
an agent that registered before its adopted engines had answered their
|
||||
health check never said they had, so the control plane kept them starting
|
||||
and routed nothing to them — on the reference cluster, after `llmgw
|
||||
restart`, the only replica of a model. The agent now reports each shard
|
||||
the registration called not ready once it is.
|
||||
- **A failover no longer replaces healthy replicas**: a standby that took
|
||||
over placed at once, before the agents had reconnected and said what runs,
|
||||
so it started a second copy of replicas it did not know of and then drained
|
||||
the healthy ones as surplus — on the reference cluster the guardrails'
|
||||
judge and a model's replica, within twenty seconds of a leader restarting.
|
||||
Placement now waits after a takeover as it does at start.
|
||||
|
||||
### Changed
|
||||
|
||||
- **A GPU placement llama.cpp says it cannot compute on is refused** when it
|
||||
warns that it found no usable GPU, even where nvidia-smi cannot be asked;
|
||||
where it can, holding no memory on its GPUs refuses it as before.
|
||||
|
||||
## v0.12.2 — 2026-10-06
|
||||
|
||||
The guardrails turned off and on as a whole, and a judge that refuses
|
||||
ordinary requests explained: the hazard it saw is told to the client and
|
||||
kept with the request, and `llmgw ctl guard judge ask` shows what it says of
|
||||
any text. Nothing to do to upgrade; the store gains a column.
|
||||
|
||||
### Added
|
||||
|
||||
- **The guardrails turn off and on as a whole**: `llmgw ctl guard off` /
|
||||
`on`, which is `guard.enabled` in the cluster's settings — in force within
|
||||
seconds, the policies and their attachments kept. `ctl status`, `ctl guard
|
||||
list` and the guardrails page say when they are off.
|
||||
- **`llmgw ctl guard judge ask TEXT`** has the judge read a text and shows
|
||||
its answer as given and as read — for finding out why ordinary requests
|
||||
are refused. Nothing of the text is kept, not even in the audit.
|
||||
|
||||
### Changed
|
||||
|
||||
- **A judge's refusal names the hazard**: the client is told `(judge:
|
||||
S6,Specialized Advice)` rather than `(judge)`, and the verdict is kept with
|
||||
the request (`guard_rule`): `ctl request`, the server's log and the
|
||||
guardrails page show it. Llama Guard 3's codes come with their names,
|
||||
S9 Indiscriminate Weapons, without a categories list.
|
||||
|
||||
## v0.12.1 — 2026-10-06
|
||||
|
||||
The guardrails' judge, served and watched without server init: a judge
|
||||
missing, served as the wrong kind of model, or not answering is said on
|
||||
`ctl status` and the console, and `llmgw ctl guard judge enable` puts it
|
||||
right. Nothing to do to upgrade.
|
||||
|
||||
### Added
|
||||
|
||||
- **`llmgw ctl guard judge status` and `enable`**: each judge model the
|
||||
policies ask, whether it is in the catalog, as a judge, with replicas
|
||||
ready and answering — and serving the missing one, Llama Guard 3 1B on
|
||||
CPUs under the name the policies give it, with `--replace` for a model of
|
||||
that name that is not a judge. No more `server init` to put a judge back.
|
||||
Also **Do → Serve the guardrails' judge** in the console.
|
||||
- **A missing or broken judge is said where it is looked for**: `llmgw ctl
|
||||
status` and the console's Overview list each judge with a problem — not in
|
||||
the catalog, not a judge, no replica ready, or failing to answer — and
|
||||
what that does to requests (refused or unjudged).
|
||||
|
||||
### Fixed
|
||||
|
||||
- **A Llama Guard file served from the mirror page is the judge**: it was
|
||||
added as a model clients use, under whatever name the form had, so the
|
||||
policies never found it and refused everything they judged
|
||||
(`guard_unavailable`). It is now added as a judge, under the name the
|
||||
policies ask.
|
||||
- **`ctl` says so when the API answers with a page**: a command a control
|
||||
plane does not know — the leader on an older release, through a standby —
|
||||
failed with "invalid character '<'"; it now says which and why.
|
||||
|
||||
## v0.12.0 — 2026-10-06
|
||||
|
||||
A minor version, because where the configuration lives changes: everything in
|
||||
|
||||
@@ -6,6 +6,7 @@ LDFLAGS := -X '$(MODULE)/internal/version.Version=$(VERSION)'
|
||||
|
||||
build:
|
||||
go build -ldflags "$(LDFLAGS)" -o bin/llmgw ./cmd/llmgw
|
||||
go build -ldflags "$(LDFLAGS)" -o bin/llmgw-kb ./cmd/llmgw-kb
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
+403
-231
File diff suppressed because it is too large.
Load diff
@@ -72,6 +72,7 @@ message ControlMessage {
|
||||
UpgradeAgent upgrade = 10;
|
||||
ControlPlanes control_planes = 11;
|
||||
Firewall firewall = 13;
|
||||
RestartAgent restart_agent = 14;
|
||||
}
|
||||
// trace is the W3C trace context (traceparent, and tracestate when there
|
||||
// is one) of the control plane's span that sent this command, so what the
|
||||
@@ -81,6 +82,12 @@ message ControlMessage {
|
||||
map<string, string> trace = 12;
|
||||
}
|
||||
|
||||
// RestartAgent has the agent start again, as llmgw restart asks of every
|
||||
// node: its engines keep serving and are taken over again.
|
||||
message RestartAgent {
|
||||
string reason = 1;
|
||||
}
|
||||
|
||||
// Firewall is whether llmgw's firewall is kept on the node — its nftables
|
||||
// table, which lets the cluster's networks alone reach the engines' ports and
|
||||
// the node exporter — sent when an agent registers and whenever the
|
||||
@@ -266,6 +273,11 @@ message RunningShard {
|
||||
// restart could not be compared with the catalog — which is exactly when the
|
||||
// catalog is most likely to have been edited.
|
||||
string profile_fp = 15;
|
||||
|
||||
// compute is where its engine said it put the model when it became ready,
|
||||
// kept by the agent across its own restarts: a control plane that takes
|
||||
// over, or restarts, learns it here.
|
||||
Compute compute = 16;
|
||||
}
|
||||
|
||||
message GPU {
|
||||
@@ -605,6 +617,18 @@ message ProvisionStatus {
|
||||
string detail = 5;
|
||||
}
|
||||
|
||||
// Compute is where an engine runs its model, as it said when it loaded it.
|
||||
message Compute {
|
||||
// backend is "CUDA", "Vulkan", "Metal" or "CPU".
|
||||
string backend = 1;
|
||||
// devices are the devices it named: "CUDA0: Quadro RTX 6000 (7.5)".
|
||||
repeated string devices = 2;
|
||||
// layers_offloaded of layers_total went to the devices; both 0 when the
|
||||
// engine did not say.
|
||||
int32 layers_offloaded = 3;
|
||||
int32 layers_total = 4;
|
||||
}
|
||||
|
||||
// ShardStatus reports a shard's lifecycle transition.
|
||||
message ShardStatus {
|
||||
string node_id = 1;
|
||||
@@ -626,6 +650,10 @@ message ShardStatus {
|
||||
// going to look.
|
||||
FetchProgress fetch = 8;
|
||||
|
||||
// compute is what the engine says it computes on, read from its own output
|
||||
// as it became ready: sent with STATE_READY when the engine says.
|
||||
Compute compute = 9;
|
||||
|
||||
enum State {
|
||||
STATE_UNSPECIFIED = 0;
|
||||
STATE_PENDING = 1;
|
||||
|
||||
@@ -0,0 +1,467 @@
|
||||
// Command llmgw-kb puts a person's documents into an llmgw knowledge base
|
||||
// from their own machine (BACKLOG F2): it reads the files there, cuts them
|
||||
// into chunks, has the gateway embed them with the person's key, and sends
|
||||
// the text and vectors — only for what changed since the last run, which a
|
||||
// ledger on the machine remembers.
|
||||
//
|
||||
// Its own binary, for Windows, macOS and Linux: it carries none of the
|
||||
// server, agent or setup code that llmgw does.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"golang.org/x/term"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/kb/extract"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/kb/kbsync"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := root().Execute(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// config is what `login` keeps: the gateway, the key, and for a personal key
|
||||
// the project it acts in.
|
||||
type config struct {
|
||||
Gateway string `json:"gateway"`
|
||||
Key string `json:"key"`
|
||||
Project string `json:"project,omitempty"`
|
||||
CA string `json:"ca,omitempty"`
|
||||
}
|
||||
|
||||
func configPath() (string, error) {
|
||||
dir, err := os.UserConfigDir()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(dir, "llmgw-kb", "config.json"), nil
|
||||
}
|
||||
|
||||
// loadConfig reads the saved configuration, with the environment over it:
|
||||
// LLMGW_URL, LLMGW_API_KEY, LLMGW_PROJECT.
|
||||
func loadConfig() (config, error) {
|
||||
var c config
|
||||
if p, err := configPath(); err == nil {
|
||||
if b, err := os.ReadFile(p); err == nil {
|
||||
if err := json.Unmarshal(b, &c); err != nil {
|
||||
return c, fmt.Errorf("%s: %w", p, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for env, dst := range map[string]*string{"LLMGW_URL": &c.Gateway, "LLMGW_API_KEY": &c.Key, "LLMGW_PROJECT": &c.Project} {
|
||||
if v := strings.TrimSpace(os.Getenv(env)); v != "" {
|
||||
*dst = v
|
||||
}
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func client(flags *globalFlags) (*kbsync.Client, error) {
|
||||
c, err := loadConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if flags.gateway != "" {
|
||||
c.Gateway = flags.gateway
|
||||
}
|
||||
if flags.project != "" {
|
||||
c.Project = flags.project
|
||||
}
|
||||
if flags.ca != "" {
|
||||
c.CA = flags.ca
|
||||
}
|
||||
if c.Gateway == "" || c.Key == "" {
|
||||
return nil, errors.New("no gateway or key: run `llmgw-kb login`, or set LLMGW_URL and LLMGW_API_KEY")
|
||||
}
|
||||
h, err := kbsync.NewHTTP(c.CA)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &kbsync.Client{Base: c.Gateway, Key: c.Key, Project: c.Project, HTTP: h}, nil
|
||||
}
|
||||
|
||||
type globalFlags struct {
|
||||
gateway, project, ca, ledger string
|
||||
}
|
||||
|
||||
func (g *globalFlags) openLedger(ctx context.Context) (*kbsync.Ledger, error) {
|
||||
p := g.ledger
|
||||
if p == "" {
|
||||
var err error
|
||||
if p, err = kbsync.DefaultLedgerPath(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return kbsync.OpenLedger(ctx, p)
|
||||
}
|
||||
|
||||
func root() *cobra.Command {
|
||||
g := &globalFlags{}
|
||||
cmd := &cobra.Command{
|
||||
Use: "llmgw-kb",
|
||||
Short: "Put your documents into an llmgw knowledge base, from this machine",
|
||||
Long: `llmgw-kb reads documents on this machine, cuts them into chunks, has the
|
||||
gateway embed them with your key, and sends the text and vectors to a
|
||||
knowledge base. Files are never uploaded. Run it as often as you like: a
|
||||
ledger on this machine remembers what was sent, and only what changed is
|
||||
read and sent again.`,
|
||||
Version: version.Version,
|
||||
SilenceUsage: true,
|
||||
SilenceErrors: true,
|
||||
}
|
||||
cmd.PersistentFlags().StringVar(&g.gateway, "gateway", "", "the gateway's address (default: from login, or LLMGW_URL)")
|
||||
cmd.PersistentFlags().StringVar(&g.project, "project", "", "the project a personal key acts in (default: from login, or LLMGW_PROJECT)")
|
||||
cmd.PersistentFlags().StringVar(&g.ca, "ca", "", "a PEM file of certificates to trust besides the system's")
|
||||
cmd.PersistentFlags().StringVar(&g.ledger, "ledger", "", "the ledger file (default: in your configuration directory)")
|
||||
cmd.AddCommand(loginCmd(g), collectionsCmd(g), syncCmd(g), statusCmd(g), historyCmd(g), formatsCmd())
|
||||
return cmd
|
||||
}
|
||||
|
||||
func loginCmd(g *globalFlags) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "login",
|
||||
Short: "Save the gateway's address and your key",
|
||||
Long: `Asks for the gateway's address and your API key, checks them, and saves them
|
||||
in your configuration directory, readable by you only. The key is typed
|
||||
without being shown, so it does not end up in your shell's history.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
c, _ := loadConfig()
|
||||
in := bufio.NewReader(os.Stdin)
|
||||
ask := func(q, def string) string {
|
||||
if def != "" {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s [%s]: ", q, def)
|
||||
} else {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s: ", q)
|
||||
}
|
||||
s, _ := in.ReadString('\n')
|
||||
if s = strings.TrimSpace(s); s == "" {
|
||||
return def
|
||||
}
|
||||
return s
|
||||
}
|
||||
c.Gateway = strings.TrimRight(ask("Gateway address (https://…)", or(g.gateway, c.Gateway)), "/")
|
||||
fmt.Fprint(cmd.OutOrStdout(), "API key: ")
|
||||
var key string
|
||||
if term.IsTerminal(int(os.Stdin.Fd())) {
|
||||
b, err := term.ReadPassword(int(os.Stdin.Fd()))
|
||||
fmt.Fprintln(cmd.OutOrStdout())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key = strings.TrimSpace(string(b))
|
||||
} else {
|
||||
key, _ = in.ReadString('\n')
|
||||
key = strings.TrimSpace(key)
|
||||
}
|
||||
if key != "" {
|
||||
c.Key = key
|
||||
}
|
||||
c.Project = ask("Project, for a personal key (empty for a project's key)", or(g.project, c.Project))
|
||||
if g.ca != "" {
|
||||
c.CA = g.ca
|
||||
}
|
||||
h, err := kbsync.NewHTTP(c.CA)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cl := &kbsync.Client{Base: c.Gateway, Key: c.Key, Project: c.Project, HTTP: h, Wait: time.Second}
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
cs, err := cl.Collections(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the gateway refused: %w", err)
|
||||
}
|
||||
p, err := configPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
b, _ := json.MarshalIndent(c, "", " ")
|
||||
if err := os.WriteFile(p, b, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "Saved in %s. This key reaches %d collection(s); `llmgw-kb collections` lists them.\n", p, len(cs))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func or(a, b string) string {
|
||||
if a != "" {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func collectionsCmd(g *globalFlags) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "collections",
|
||||
Short: "List the collections your key may read, and which it may add to",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
c, err := client(g)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cs, err := c.Collections(cmd.Context())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(cs) == 0 {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "No collections. An administrator creates them and grants access (llmgw ctl kb).")
|
||||
return nil
|
||||
}
|
||||
tw := tabwriter.NewWriter(cmd.OutOrStdout(), 2, 4, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, "NAME\tACCESS\tOWNER\tEMBEDDED BY")
|
||||
for _, x := range cs {
|
||||
access := "read"
|
||||
if x.Writable {
|
||||
access = "read, write"
|
||||
}
|
||||
fmt.Fprintf(tw, "%s\t%s\t%s\t%s (%d)\n", x.Name, access, or(x.OwnerName, x.Owner), x.EmbedModel, x.Dims)
|
||||
}
|
||||
return tw.Flush()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func syncCmd(g *globalFlags) *cobra.Command {
|
||||
var o kbsync.Options
|
||||
var verbose bool
|
||||
var ocr string
|
||||
var maxMiB int64
|
||||
cmd := &cobra.Command{
|
||||
Use: "sync COLLECTION PATH...",
|
||||
Short: "Send a collection what changed in these folders and files",
|
||||
Long: `Reads the files under each PATH that the collection does not already have as
|
||||
they are, and sends their text and vectors. Run it again whenever you like;
|
||||
unchanged files are not even read.
|
||||
|
||||
Documents are named by their path under PATH, with forward slashes; with
|
||||
several PATHs, each folder's own name comes first.
|
||||
|
||||
--dry-run reads and cuts everything, sends nothing, and says what would
|
||||
leave this machine
|
||||
--delete removes documents this machine sent whose files are gone
|
||||
(only under these PATHs; nothing anybody else sent)
|
||||
--retry reads again files skipped before (after installing LibreOffice
|
||||
or Tesseract, say)
|
||||
|
||||
Without --include, every file of a known format is read (llmgw-kb formats);
|
||||
hidden files, version control folders and Office lock files are not.`,
|
||||
Args: cobra.MinimumNArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
c, err := client(g)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt)
|
||||
defer stop()
|
||||
l, err := g.openLedger(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer l.Close()
|
||||
o.Collection, o.Roots = args[0], args[1:]
|
||||
o.Extract = extract.Options{Helpers: extract.FindHelpers(), OCRLanguages: ocr, MaxBytes: maxMiB << 20}
|
||||
out := cmd.OutOrStdout()
|
||||
c.Waiting = func(why string, d time.Duration) {
|
||||
fmt.Fprintf(out, " .. waiting %s: %s\n", d, why)
|
||||
}
|
||||
start := time.Now()
|
||||
rep, err := kbsync.Sync(ctx, c, l, o, func(e kbsync.Event) {
|
||||
switch e.Kind {
|
||||
case "unchanged":
|
||||
if !verbose {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(out, " = %s\n", e.Name)
|
||||
case "sent", "would-send":
|
||||
fmt.Fprintf(out, " %-8s %s (%d chunks, %s)\n", label(e.Kind), e.Name, e.Chunks, size(e.Bytes))
|
||||
default:
|
||||
line := fmt.Sprintf(" %-8s %s", label(e.Kind), e.Name)
|
||||
if e.Detail != "" {
|
||||
line += ": " + e.Detail
|
||||
}
|
||||
fmt.Fprintln(out, line)
|
||||
}
|
||||
})
|
||||
if rep.Collection.Name != "" {
|
||||
verb := "sent"
|
||||
if o.DryRun {
|
||||
verb = "would send"
|
||||
}
|
||||
fmt.Fprintf(out, "\n%s: %d files; %s %d (%d chunks, %s), %d unchanged, %d skipped, %d failed, %d deleted, in %s\n",
|
||||
rep.Collection.Name, rep.Files, verb, rep.Sent, rep.Chunks, size(rep.Bytes), rep.Unchanged,
|
||||
rep.Skipped, rep.Failed, rep.Deleted, time.Since(start).Round(time.Second))
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rep.Failed > 0 {
|
||||
return fmt.Errorf("%d document(s) failed; run again to retry them", rep.Failed)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
f := cmd.Flags()
|
||||
f.StringSliceVar(&o.Include, "include", nil, "read only files matching these globs (\"*.pdf,docs/**/*.md\")")
|
||||
f.StringSliceVar(&o.Exclude, "exclude", nil, "never read files matching these globs")
|
||||
f.BoolVar(&o.Hidden, "hidden", false, "read hidden files and folders too")
|
||||
f.BoolVar(&o.DryRun, "dry-run", false, "send nothing; say what would be sent")
|
||||
f.BoolVar(&o.Delete, "delete", false, "remove documents this machine sent whose files are gone")
|
||||
f.BoolVar(&o.Retry, "retry", false, "read again files skipped before")
|
||||
f.IntVar(&o.Workers, "workers", 4, "files read and sent at once")
|
||||
f.StringVar(&ocr, "ocr-languages", "tur+eng", "Tesseract's languages for images and scanned pages")
|
||||
f.Int64Var(&maxMiB, "max-size", 256, "skip files larger than this many MiB")
|
||||
f.BoolVarP(&verbose, "verbose", "v", false, "list unchanged files too")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func label(kind string) string {
|
||||
switch kind {
|
||||
case "would-send":
|
||||
return "send"
|
||||
case "would-delete":
|
||||
return "delete"
|
||||
}
|
||||
return kind
|
||||
}
|
||||
|
||||
func size(b int64) string {
|
||||
switch {
|
||||
case b >= 1<<20:
|
||||
return fmt.Sprintf("%.1f MiB", float64(b)/(1<<20))
|
||||
case b >= 1<<10:
|
||||
return fmt.Sprintf("%.0f KiB", float64(b)/(1<<10))
|
||||
}
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
|
||||
func statusCmd(g *globalFlags) *cobra.Command {
|
||||
var all bool
|
||||
return &cobra.Command{
|
||||
Use: "status COLLECTION",
|
||||
Short: "Show what this machine has sent to a collection, and what it skipped",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
c, err := client(g)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m, err := c.Manifest(cmd.Context(), args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
l, err := g.openLedger(cmd.Context())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer l.Close()
|
||||
es, err := l.Entries(cmd.Context(), strings.TrimRight(c.Base, "/"), m.Collection.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
counts := map[kbsync.State]int{}
|
||||
tw := tabwriter.NewWriter(cmd.OutOrStdout(), 2, 4, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, "STATE\tDOCUMENT\tCHUNKS\tWHEN\tWHY")
|
||||
for _, e := range es {
|
||||
counts[e.State]++
|
||||
if e.State == kbsync.Sent && !all {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(tw, "%s\t%s\t%d\t%s\t%s\n", e.State, e.Name, e.Chunks, e.At.Format("2006-01-02 15:04"), e.Reason)
|
||||
}
|
||||
_ = tw.Flush()
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "\n%s holds %d documents; from this machine %d sent, %d skipped, %d failed.",
|
||||
m.Collection.Name, len(m.Documents), counts[kbsync.Sent], counts[kbsync.Skipped], counts[kbsync.Failed])
|
||||
if !all {
|
||||
fmt.Fprint(cmd.OutOrStdout(), " (--all lists the sent ones too)")
|
||||
}
|
||||
fmt.Fprintln(cmd.OutOrStdout())
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func historyCmd(g *globalFlags) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "history",
|
||||
Short: "List the latest syncs from this machine",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
l, err := g.openLedger(cmd.Context())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer l.Close()
|
||||
runs, err := l.Runs(cmd.Context(), 20)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tw := tabwriter.NewWriter(cmd.OutOrStdout(), 2, 4, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, "STARTED\tTOOK\tCOLLECTION\tSENT\tUNCHANGED\tSKIPPED\tFAILED\tDELETED")
|
||||
for _, r := range runs {
|
||||
took := "unfinished"
|
||||
if !r.Finished.IsZero() {
|
||||
took = r.Finished.Sub(r.Started).Round(time.Second).String()
|
||||
}
|
||||
fmt.Fprintf(tw, "%s\t%s\t%s\t%d\t%d\t%d\t%d\t%d\n", r.Started.Format("2006-01-02 15:04"), took,
|
||||
r.Collection, r.Sent, r.Unchanged, r.Skipped, r.Failed, r.Delete)
|
||||
}
|
||||
return tw.Flush()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func formatsCmd() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "formats",
|
||||
Short: "List the formats read, and the helpers found on this machine",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
builtIn, helper := extract.Supported()
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "Read here:\n%s\n\n", wrap(builtIn))
|
||||
fmt.Fprintf(out, "Read with a helper program:\n%s\n\n", wrap(helper))
|
||||
fmt.Fprintln(out, "Helpers:")
|
||||
for _, l := range extract.FindHelpers().Describe() {
|
||||
fmt.Fprintln(out, " "+l)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func wrap(words []string) string {
|
||||
var b strings.Builder
|
||||
line := " "
|
||||
for _, w := range words {
|
||||
if len(line)+len(w) > 78 {
|
||||
b.WriteString(strings.TrimRight(line, " ") + "\n")
|
||||
line = " "
|
||||
}
|
||||
line += w + " "
|
||||
}
|
||||
b.WriteString(strings.TrimRight(line, " "))
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The client goes onto employees' laptops: none of the server, agent, store
|
||||
// or setup code may come with it, and it must build where llmgw does not.
|
||||
func TestTheClientCarriesNoServerCode(t *testing.T) {
|
||||
out, err := exec.Command("go", "list", "-deps", ".").Output()
|
||||
if err != nil {
|
||||
t.Skipf("go list: %v", err)
|
||||
}
|
||||
for _, dep := range strings.Fields(string(out)) {
|
||||
for _, banned := range []string{"/internal/store", "/internal/router", "/internal/agent", "/internal/control",
|
||||
"/internal/cli", "/internal/guard", "github.com/jackc/pgx"} {
|
||||
if strings.Contains(dep, banned) {
|
||||
t.Errorf("llmgw-kb depends on %s", dep)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -39,7 +39,7 @@ func (d Doc) Source() string {
|
||||
// than with either reader, because the console's manual and the gateway's
|
||||
// docs portal both list them, and two lists drift.
|
||||
var docOrder = []string{"console", "quick-install", "install", "node-onboarding", "onboarding",
|
||||
"clients", "guardrails", "choosing-models", "fabrics", "facility-aware-placement", "data",
|
||||
"clients", "guardrails", "knowledge", "choosing-models", "fabrics", "facility-aware-placement", "data",
|
||||
"reference", "development"}
|
||||
|
||||
// Manual is every document, in reading order: those docOrder names, then any
|
||||
|
||||
+928
-2
File diff suppressed because it is too large.
Load diff
@@ -164,6 +164,15 @@ which): requests, tokens, errors and spend.
|
||||
team or project is guaranteed and may at most hold, and what it holds now.
|
||||
**A** adds one; on a row, change or remove it.
|
||||
|
||||
**Knowledge** lists the knowledge-base collections: each one's owner, the
|
||||
model that embeds it, how many documents and chunks it holds and who else it
|
||||
is granted to. **A** makes one. On a row: its **Documents**, **Grant…** a
|
||||
project, team, organization, person or key read or write access, **Revoke**
|
||||
a grant, **Answer from it by default…** for a part of the tree, **Try a
|
||||
search…** as an administrator, or **Delete…** it. People
|
||||
fill collections from their own machines with `llmgw-kb`
|
||||
([knowledge.md](knowledge.md)). Off on a SQLite store, and says why.
|
||||
|
||||
**Mirror** is what the model mirror holds, one row for each version of each
|
||||
model — a file, or the set of files a model of several is: shards, a vision
|
||||
projector, a config and tokenizer — with how far its download has got and
|
||||
|
||||
+124
-16
@@ -4,7 +4,7 @@ Written for the operator deciding what the gateway lets through: what a
|
||||
policy can check, what it can do, and how one is attached to a team.
|
||||
|
||||
`llmgw server init` writes a `baseline` policy that applies to every
|
||||
request, with Llama Guard 3 1B as its judge on a CPU; see
|
||||
request, with Qwen3Guard-Gen 0.6B as its judge on CPUs; see
|
||||
[install.md](install.md#2-the-control-plane) for what it checks. The
|
||||
policies are part of the cluster's settings, kept in the store: `llmgw ctl
|
||||
config edit` (or the console's Settings) changes them, and the change is in
|
||||
@@ -15,6 +15,19 @@ checked and nothing costs anything. With one, every request is judged once it is
|
||||
within its limits, with its stored conversation spliced in, and before it is
|
||||
sent to a model; and the answer is judged before the client sees it.
|
||||
|
||||
## Turning them off and on
|
||||
|
||||
```
|
||||
llmgw ctl guard off --note "why" # no request is checked, whatever the policies say
|
||||
llmgw ctl guard on # every policy checks again, as it was
|
||||
```
|
||||
|
||||
That is `guard.enabled` in the cluster's settings, in force on every control
|
||||
plane within seconds without a restart. The policies and where they are
|
||||
attached are kept. While off, `llmgw ctl status`, `llmgw ctl guard list` and
|
||||
the guardrails page say so. To stop one policy rather than all, give it
|
||||
`mode: off`, or `mode: shadow` to have it record what it would do.
|
||||
|
||||
## A policy
|
||||
|
||||
```yaml
|
||||
@@ -101,44 +114,136 @@ guard:
|
||||
|
||||
## A judge model
|
||||
|
||||
Patterns find what has a shape. For what does not — violence, self-harm,
|
||||
abuse, whatever a moderation model was trained on — a policy can have a
|
||||
**judge model on the cluster** read the text:
|
||||
Patterns find what has a shape. For what does not — violence, crime,
|
||||
sexual content, self-harm, jailbreaks, whatever a moderation model was
|
||||
trained on — a policy can have a **judge model on the cluster** read the
|
||||
text.
|
||||
|
||||
The judge must be in the catalog under the name the policy gives, as a
|
||||
judge (`role: guard`), with a replica ready. When it is not, what the
|
||||
policy judges is refused with `guard_unavailable` (`fail: closed`) or served
|
||||
unjudged (`fail: open`), and `llmgw ctl status` and the console's Overview
|
||||
say which judge and why.
|
||||
|
||||
### Which judge
|
||||
|
||||
| judge | runs on | size | |
|
||||
|---|---|---|---|
|
||||
| `qwen3guard-0.6b` | CPUs | 0.8 GB | the default: about 2 s a check on two threads |
|
||||
| `qwen3guard-4b` | a GPU | 2.5 GB | for a cluster with a card to spare |
|
||||
| `qwen3guard-8b` | a GPU | 4.7 GB | |
|
||||
| `llama-guard-3-1b` | CPUs | 0.9 GB | the earlier default: refuses ordinary coding requests |
|
||||
| `llama-guard-3-8b` | a GPU | 4.6 GB | |
|
||||
|
||||
Measured on the reference cluster, as the judge's probability of "unsafe"
|
||||
for ordinary coding requests against harmful ones, Llama Guard 3 1B could
|
||||
not tell them apart: "continue" 0.51, "continue the code above" 0.89, "how
|
||||
do I kill all child processes in bash" 0.92, against 0.97 to 1.00 for
|
||||
harmful requests, at any quantisation. Qwen3Guard-Gen 0.6B gave the
|
||||
ordinary ones 0.15 at the most and the harmful ones 1.00. Its verdicts are
|
||||
safe, unsafe and **controversial**; controversial counts as safe unless the
|
||||
policy says `controversial: unsafe`.
|
||||
|
||||
A policy that names one of these is enough: the leader fetches its weights
|
||||
into the mirror and serves it as a judge, and serves it again if it is
|
||||
removed from the catalog. Another judge is served by hand with `--repo`.
|
||||
|
||||
```
|
||||
llmgw ctl guard judge status # each judge the policies ask, and what is wrong with it
|
||||
llmgw ctl guard judge enable # Qwen3Guard-Gen 0.6B on CPUs, or the judge the policies ask that is missing
|
||||
llmgw ctl guard judge enable --gpu # Qwen3Guard-Gen 4B on a GPU
|
||||
llmgw ctl guard judge use qwen3guard-4b qwen3guard-0.6b # the order every policy asks them in
|
||||
llmgw ctl guard judge enable --replace # when a model of that name is there, but not as a judge
|
||||
```
|
||||
|
||||
or **Do → Serve the guardrails' judge** in the console.
|
||||
|
||||
### A GPU and CPUs together
|
||||
|
||||
`judge.models` is a list asked in turn: the first judge that answers gives
|
||||
the verdict. The next is asked when one has no replica serving, fails, or
|
||||
does not answer within `latency_budget`; a judge with no replica at all is
|
||||
passed over at once. A judge on a GPU first and one on CPUs after it is
|
||||
the GPU's speed while it has its card, and the CPUs' when it does not.
|
||||
|
||||
```yaml
|
||||
judge:
|
||||
models: [qwen3guard-4b, qwen3guard-0.6b]
|
||||
input: block
|
||||
output: block
|
||||
latency_budget: 5s # each judge's wait
|
||||
```
|
||||
|
||||
A judge on a GPU is served as a `standard` model, on one card: it takes a
|
||||
card only from what may give one up, and waits while every card is busy,
|
||||
the judge on CPUs standing in. A judge on CPUs is `critical`, with two
|
||||
replicas, because every judged request waits on one. `llmgw ctl guard judge
|
||||
status` says, for each judge, which is asked when it gives no verdict.
|
||||
|
||||
### When it refuses ordinary requests
|
||||
|
||||
A refusal by a judge names the hazard it saw — the client is told `refused
|
||||
by the gateway's policy (judge: Violent)` — and the same is kept with the
|
||||
request: `llmgw ctl request <id>`, the server's log (`guard_rule`) and the
|
||||
guardrails page show it. To see what the judge says of any text, without
|
||||
anything being kept:
|
||||
|
||||
```
|
||||
llmgw ctl guard judge ask 'How do I sort a list in Python?'
|
||||
# qwen3guard-0.6b: safe, 1900 ms
|
||||
# it answered: "Safety: Safe\nCategories: None"
|
||||
```
|
||||
|
||||
A judge that says `unsafe` to ordinary text is usually Llama Guard 3 1B
|
||||
(`llmgw ctl guard judge use qwen3guard-0.6b`), the wrong weights (another
|
||||
repository's, without the judge's template: `judge enable --replace` serves
|
||||
the right ones), or a policy counting every category: give its judge
|
||||
`categories`. The baseline counts Violent, Non-violent Illegal Acts, Sexual
|
||||
Content or Sexual Acts, Suicide & Self-Harm and Jailbreak; PII is the
|
||||
patterns', and copyright, unethical acts and politically sensitive topics
|
||||
refuse too much ordinary work. While finding out, `judge.input: flag`
|
||||
records instead of refusing.
|
||||
|
||||
A judge's file served from the mirror page — Llama Guard, Qwen3Guard,
|
||||
ShieldGemma — is served as a judge too, under the name the policies ask.
|
||||
By hand, the catalog entry is:
|
||||
|
||||
```yaml
|
||||
# a catalog model (server.yaml models:, or ctl model add)
|
||||
- name: llama-guard-3-1b
|
||||
source: http://mirror:8081/llama-guard-3-1b/…/Llama-Guard-3-1B-Q4_K_M.gguf
|
||||
- name: qwen3guard-0.6b
|
||||
source: http://mirror:8081/qwen3guard-0.6b/…/Qwen3Guard-Gen-0.6B.Q8_0.gguf
|
||||
role: guard
|
||||
guard:
|
||||
format: llamaguard3 # llamaguard3, shieldgemma or json
|
||||
categories: [Violent Crimes, Non-Violent Crimes, Sex-Related Crimes, …] # names for S1, S2, …
|
||||
format: qwen3guard # qwen3guard, llamaguard3, shieldgemma or json
|
||||
scaling: {min: 2, max: 2}
|
||||
profiles: [{name: cpu, engine: llamacpp}]
|
||||
|
||||
# in a guard policy
|
||||
judge:
|
||||
model: llama-guard-3-1b
|
||||
models: [qwen3guard-0.6b] # or model: one judge
|
||||
input: block # flag or block; empty does not ask about input
|
||||
output: flag
|
||||
categories: [S1, S10] # only these count; empty: any unsafe verdict
|
||||
latency_budget: 250ms # then the policy's fail: open serves (flagged), closed refuses
|
||||
categories: [Violent, Jailbreak] # only these count; empty: any unsafe verdict
|
||||
controversial: safe # or unsafe
|
||||
latency_budget: 250ms # each judge's wait; after the last, fail: open serves (flagged), closed refuses
|
||||
```
|
||||
|
||||
A model with `role: guard` is the guardrails' alone:
|
||||
|
||||
- it is not in `/v1/models` or the chat page, and is refused (404) to
|
||||
every key, as a name and as a fallback;
|
||||
- it is `critical` by default and must always run — on demand is refused.
|
||||
Give it two replicas on different nodes, because every judged request
|
||||
waits on it.
|
||||
- it must always run — on demand is refused. One on CPUs is `critical`
|
||||
and should have two replicas on different nodes, because every judged
|
||||
request waits on it.
|
||||
|
||||
The gateway's calls to it go past admission, accounting and the guardrails
|
||||
themselves.
|
||||
|
||||
- **Formats:**
|
||||
- `llamaguard3` reads Meta's "safe" / "unsafe
|
||||
S1,S10".
|
||||
- `qwen3guard` asks in Qwen3Guard's own words and reads "Safety: Safe |
|
||||
Unsafe | Controversial" and its "Categories:".
|
||||
- `llamaguard3` reads Meta's "safe" / "unsafe\nS1,S10"; give the catalog
|
||||
entry `categories` to have S1 recorded as its name.
|
||||
- `shieldgemma` asks Google's model whether the text breaks the listed
|
||||
principles (`categories`, each a sentence) and reads Yes or No.
|
||||
- `json` expects `{"safe": false, "categories": [...]}`.
|
||||
@@ -148,6 +253,9 @@ S1,S10".
|
||||
is read every 600 characters and at the end, so text between two
|
||||
readings has gone out already; `stream: buffer_all` holds it until it
|
||||
is read.
|
||||
- **A long text** is read by its start and its end (about 6000
|
||||
characters), so a message with a file pasted into it fits the judge's
|
||||
context rather than overflowing it and going unjudged.
|
||||
- **Cache:** the same text is not asked about twice within five minutes.
|
||||
|
||||
## Where a policy applies
|
||||
|
||||
+40
-2
@@ -129,8 +129,9 @@ That is the whole step. In order, it:
|
||||
`nvidia-smi` finds any, on CPU otherwise — and waits for it to install its
|
||||
engine
|
||||
9. closes llmgw's internal ports to everything but the cluster's networks
|
||||
10. fetches Llama Guard 3 1B, the guardrails' judge, and runs two replicas
|
||||
on the CPUs of nodes with room — this one, or the first that join
|
||||
10. fetches Qwen3Guard-Gen 0.6B, the guardrails' judge, and runs two
|
||||
replicas on the CPUs of nodes with room — this one, or the first that
|
||||
join ([guardrails.md](guardrails.md#a-judge-model) has one on a GPU too)
|
||||
11. fetches Qwen2.5 0.5B into the mirror and serves it on this node, so the
|
||||
first thing anyone opens `/chat` to find is a model that answers. A
|
||||
machine without room for it is told so within two minutes, and why.
|
||||
@@ -2294,6 +2295,43 @@ A cluster set up online becomes offline the same way: `bundle import` on the
|
||||
mirror and control plane, then the `offline:` block in `server.yaml` and a
|
||||
restart.
|
||||
|
||||
## Stopping and starting the system
|
||||
|
||||
From a control plane:
|
||||
|
||||
```
|
||||
llmgw status # does it serve; its control planes, nodes, models, store
|
||||
llmgw stop --note "maintenance" # every model taken down, every request answered 503
|
||||
llmgw start # every model back as its catalog entry asks
|
||||
sudo llmgw restart # every agent and control plane started again, engines serving throughout
|
||||
```
|
||||
|
||||
`llmgw stop` frees every GPU and answers requests for a model with 503
|
||||
(`system_stopped`, with Retry-After) until `llmgw start`. The control planes,
|
||||
agents and mirror keep running, so the console, the admin pages and `llmgw
|
||||
start` keep working, and no model's definition changes. It is the cluster's
|
||||
setting `serving.stopped`, so a restart does not undo it, and it is in the
|
||||
audit with its note. `--wait` waits until every replica is down, or every
|
||||
model has its replicas again.
|
||||
|
||||
`llmgw restart` restarts every node's agent, then the other control planes
|
||||
over ssh (`--ssh-user`, the one sudo was run by), then this one — the leader
|
||||
last, so it hands over to a standby already restarted. Engines keep serving
|
||||
and are taken over again; nothing is placed anew.
|
||||
|
||||
One machine's own services are `llmgw service`:
|
||||
|
||||
```
|
||||
sudo llmgw service status
|
||||
sudo llmgw service stop # agent, control plane, mirror — whichever this machine has
|
||||
sudo llmgw service start
|
||||
sudo llmgw service restart agent # or name one: server, agent, mirror
|
||||
```
|
||||
|
||||
Stopping a node's agent leaves its engines serving — the agent takes them
|
||||
over again when it starts — unless `llmgw service stop --engines` stops them
|
||||
too, and the control plane then places their replicas elsewhere.
|
||||
|
||||
## Removing llmgw from a machine
|
||||
|
||||
```
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
# Knowledge bases
|
||||
|
||||
A knowledge base is a collection of a company's documents that the cluster
|
||||
can answer from. People put their documents into it from their own machines
|
||||
with `llmgw-kb`. The tool reads the files on that machine, cuts them into
|
||||
passages ("chunks"), has the gateway embed each passage with the person's own
|
||||
key, and sends the text and the vectors. The files themselves never leave
|
||||
the machine.
|
||||
|
||||
## Why retrieval, and not training
|
||||
|
||||
Retrieval is the tool for *knowledge*: policies, procedures, contracts. The
|
||||
answer can name its source, a changed document is sent again in minutes, and
|
||||
a passage only ever reaches people allowed to read its collection.
|
||||
|
||||
Training a model on the documents (LoRA) changes how it *writes*, not what it
|
||||
reliably *knows*. It has no access control: whatever goes into the weights
|
||||
is available to everyone who may use them.
|
||||
|
||||
## What it needs
|
||||
|
||||
- **The PostgreSQL store with pgvector.** On a SQLite store knowledge bases
|
||||
are off, and `llmgw ctl kb list` and the console say why.
|
||||
- `llmgw server init` and the high-availability database setup install
|
||||
pgvector and enable it.
|
||||
- A cluster set up before this build needs it by hand, once:
|
||||
1. Install the package on every database machine:
|
||||
`sudo apt-get install postgresql-16-pgvector` (use the server's
|
||||
version number).
|
||||
2. On the leader, enable it in llmgw's database:
|
||||
`sudo -u postgres psql -d llmgw -c 'CREATE EXTENSION vector'`.
|
||||
3. Restart the control planes: `llmgw restart`.
|
||||
- **An embedding model in the catalog**, with `capabilities.embeddings`.
|
||||
Two good choices handle Turkish and run on CPUs: `BAAI/bge-m3` (1024
|
||||
dimensions) and `Qwen/Qwen3-Embedding-0.6B`.
|
||||
|
||||
Add it as any model is added (**Serve a model** in the console, or the
|
||||
catalog), with its entry saying that it embeds:
|
||||
|
||||
```yaml
|
||||
capabilities:
|
||||
embeddings: true
|
||||
```
|
||||
|
||||
A collection stays with the model it was made with: vectors from two
|
||||
models cannot be compared.
|
||||
|
||||
## For administrators
|
||||
|
||||
### Make a collection
|
||||
|
||||
A collection belongs to one part of the tenancy tree: an organization, a
|
||||
team, a project or a person. Whoever is in the owner, at any level below it,
|
||||
may read it and add to it.
|
||||
|
||||
```sh
|
||||
llmgw ctl kb create team:Acme/hr policies bge-m3
|
||||
llmgw ctl kb create user:USER-ID notes bge-m3 --org Acme
|
||||
```
|
||||
|
||||
The model's dimension is asked of the model itself, so the model has to be
|
||||
able to serve when the collection is made. `--dims 1024` gives it instead.
|
||||
|
||||
In the console: **Knowledge**, then **A**.
|
||||
|
||||
### Who may read and add
|
||||
|
||||
The owner, and everything below it, may read and add. A **grant** gives the
|
||||
same to another part of the same organization, to a person, or to one key:
|
||||
|
||||
```sh
|
||||
llmgw ctl kb grant Acme/policies project:Acme/sales/crm # read
|
||||
llmgw ctl kb grant Acme/policies team:Acme/legal --write # read and add
|
||||
llmgw ctl kb revoke Acme/policies project:Acme/sales/crm
|
||||
```
|
||||
|
||||
How access works:
|
||||
|
||||
- **Grants add up down the tree.** A grant to a team reaches its projects
|
||||
and their keys.
|
||||
- **Nothing crosses organizations.** A grant to another organization's
|
||||
team, project, key or person is refused.
|
||||
- **A person's own key** (a key with no project) reads nothing by itself.
|
||||
The person names the project they act in (`llmgw-kb login` asks for it,
|
||||
or `--project`) and must be a member of it. Joining a second team
|
||||
therefore does not quietly widen what every key of theirs can read.
|
||||
- **No access looks like no collection.** A collection a key may not read
|
||||
is answered exactly as one that does not exist, so even its name does not
|
||||
leak.
|
||||
|
||||
### Answer from collections by default
|
||||
|
||||
Clients that cannot add a header still get answers from the documents when
|
||||
their part of the tree has a default:
|
||||
|
||||
```sh
|
||||
llmgw ctl kb default team:Acme/hr Acme/policies Acme/forms
|
||||
llmgw ctl kb default key:KEY-ID Acme/policies
|
||||
llmgw ctl kb default team:Acme/hr --clear
|
||||
```
|
||||
|
||||
- **Which default applies.** The narrowest one: a key's over its project's,
|
||||
a project's over its team's, a team's over its organization's.
|
||||
- **A request can override it.** A request that names collections uses
|
||||
those instead, and `X-Llmgw-Knowledge: none` turns the default off for
|
||||
that request.
|
||||
- **A default never refuses a request.** A collection the caller may not
|
||||
read is left out for them. If the search fails (the embedding model is
|
||||
down, say), the request is answered without passages and the failure is
|
||||
logged.
|
||||
- **Changes take effect within 30 seconds.**
|
||||
|
||||
In the console: **Knowledge**, Enter on a collection, then **Answer from it
|
||||
by default…**.
|
||||
|
||||
### Look after it
|
||||
|
||||
```sh
|
||||
llmgw ctl kb list # collections, owners, documents, grants
|
||||
llmgw ctl kb docs Acme/policies # its documents, who added each and when
|
||||
llmgw ctl kb rm Acme/policies old/form.pdf # one document out
|
||||
llmgw ctl kb search Acme/policies how many days of annual leave
|
||||
llmgw ctl kb delete Acme/policies # the collection, its documents and grants
|
||||
```
|
||||
|
||||
Rules that apply:
|
||||
|
||||
- A project, team or organization that owns a collection is not deleted
|
||||
until its collections are. This is the same rule as for one with members.
|
||||
- A person's own collections go with their account (`users delete`,
|
||||
`users forget`).
|
||||
- Every document added or removed is in the audit log (`llmgw ctl audit`),
|
||||
by name. The text is never recorded.
|
||||
|
||||
## For everyone: `llmgw-kb`
|
||||
|
||||
### Get it
|
||||
|
||||
Download `llmgw-kb` for your system from the release page, next to `llmgw`:
|
||||
|
||||
| System | File |
|
||||
|---|---|
|
||||
| Windows (Intel/AMD) | `llmgw-kb-VERSION-windows-amd64.exe` |
|
||||
| Windows (ARM) | `llmgw-kb-VERSION-windows-arm64.exe` |
|
||||
| macOS (Apple silicon) | `llmgw-kb-VERSION-darwin-arm64` |
|
||||
| macOS (Intel) | `llmgw-kb-VERSION-darwin-amd64` |
|
||||
| Linux | `llmgw-kb-VERSION-linux-amd64` / `-arm64` |
|
||||
|
||||
Rename it to `llmgw-kb` (`llmgw-kb.exe` on Windows) and put it somewhere on
|
||||
your PATH. You do not need to install anything else.
|
||||
|
||||
On macOS, a downloaded program that is not signed is stopped the first time.
|
||||
Either right-click it and choose Open, or run
|
||||
`xattr -d com.apple.quarantine llmgw-kb`.
|
||||
|
||||
### Log in once
|
||||
|
||||
```sh
|
||||
llmgw-kb login
|
||||
```
|
||||
|
||||
It asks for three things, checks them, and saves them where only you can
|
||||
read them:
|
||||
|
||||
- **The gateway's address.**
|
||||
- **Your API key.** It is typed without being shown.
|
||||
- **A project.** Needed only for a personal key.
|
||||
|
||||
Instead of logging in you can set `LLMGW_URL`, `LLMGW_API_KEY` and
|
||||
`LLMGW_PROJECT`.
|
||||
|
||||
```sh
|
||||
llmgw-kb collections # what your key may read, and which you may add to
|
||||
```
|
||||
|
||||
### Send your documents
|
||||
|
||||
```sh
|
||||
llmgw-kb sync policies ~/Documents/HR
|
||||
llmgw-kb sync policies "C:\Users\ayse\Documents\İK" --include "*.pdf,*.docx"
|
||||
llmgw-kb sync policies ./handbook ./forms --exclude "drafts/**"
|
||||
```
|
||||
|
||||
Run it as often as you like. The first run sends everything. Later runs
|
||||
send only what changed, and do not even read files whose size and time have
|
||||
not changed.
|
||||
|
||||
| Option | What it does |
|
||||
|---|---|
|
||||
| `--dry-run` | Reads and cuts everything, sends nothing, and lists what would leave the machine and how much. |
|
||||
| `--delete` | Also removes documents whose files are gone. Only those this machine sent, from these folders; never anything someone else sent. |
|
||||
| `--retry` | Reads again files skipped before, for example after installing LibreOffice. |
|
||||
| `--include`, `--exclude` | Glob patterns. `*.pdf` matches a file name; `reports/**/*.xlsx` matches a path. |
|
||||
| `--hidden` | Reads hidden files and folders too (left out by default). |
|
||||
| `-v` | Lists unchanged files too. |
|
||||
|
||||
Documents are named by their path under the folder, with forward slashes,
|
||||
on every system. With several folders, each folder's own name comes first.
|
||||
|
||||
```sh
|
||||
llmgw-kb status policies # what this machine sent and skipped, and why
|
||||
llmgw-kb history # the latest runs
|
||||
llmgw-kb formats # what can be read here
|
||||
```
|
||||
|
||||
### What it reads
|
||||
|
||||
| Read here | Formats |
|
||||
|---|---|
|
||||
| Documents | PDF (text), Word (`.docx`), Excel (`.xlsx`), PowerPoint (`.pptx`, with speaker notes), OpenDocument (`.odt`, `.ods`, `.odp`), RTF, EPUB |
|
||||
| Web and mail | HTML, MHTML, e-mail (`.eml`, with its attachments) |
|
||||
| Text | Markdown, plain text, CSV, JSON, YAML, XML, subtitles, notebooks (`.ipynb`), source code |
|
||||
| Archives | ZIP, with every file inside it read in turn |
|
||||
|
||||
| Read with a helper on your machine | Helper |
|
||||
|---|---|
|
||||
| Word, Excel and PowerPoint 97–2003 (`.doc`, `.xls`, `.ppt`), Pages, Numbers, Keynote and others | [LibreOffice](https://www.libreoffice.org) |
|
||||
| Images and scanned PDFs | [Tesseract](https://tesseract-ocr.github.io) (with the Turkish language data), plus Poppler's `pdftoppm` for PDFs |
|
||||
| PDFs the built-in reader cannot parse | Poppler's `pdftotext` |
|
||||
|
||||
`llmgw-kb formats` says which helpers it found. Text in UTF-8, UTF-16, or
|
||||
the Windows Turkish and Western code pages is read correctly.
|
||||
|
||||
A file that cannot be read is skipped with the reason, and the run goes on.
|
||||
Reasons include a locked Office file, a format with no helper, and a
|
||||
scanned page with no OCR.
|
||||
|
||||
### What leaves your machine
|
||||
|
||||
For each new or changed document, two things leave the machine, both over
|
||||
the gateway's TLS connection:
|
||||
|
||||
- **The passages, to be embedded.** They go through `/v1/embeddings` like
|
||||
any request of yours. Your key's limits, budget and guardrails apply.
|
||||
- **The document itself.** Its name, its file's SHA-256 digest and size,
|
||||
and the passages with their vectors.
|
||||
|
||||
The file itself is never sent. `--dry-run` shows exactly what would be.
|
||||
|
||||
### The ledger
|
||||
|
||||
`llmgw-kb` keeps a small SQLite database of what it did with every
|
||||
document: the file it came from, its size, time and digest, and whether it
|
||||
was sent, skipped or failed, with the reason.
|
||||
|
||||
| System | Where it is |
|
||||
|---|---|
|
||||
| Windows | `%AppData%\llmgw-kb\ledger.db` |
|
||||
| macOS | `~/Library/Application Support/llmgw-kb/ledger.db` |
|
||||
| Linux | `~/.config/llmgw-kb/ledger.db` |
|
||||
|
||||
Deleting the ledger is safe: the next run hashes every file once, finds what
|
||||
the collection already has, and sends nothing that is unchanged. `--ledger`
|
||||
uses another file.
|
||||
|
||||
## Asking questions of a collection
|
||||
|
||||
A chat request names the collections to answer from in a header. Any client
|
||||
that can add a header can do it: the OpenAI and Anthropic SDKs, Continue, or
|
||||
curl.
|
||||
|
||||
```sh
|
||||
curl https://llm.example.com/v1/chat/completions \
|
||||
-H "Authorization: Bearer $LLMGW_API_KEY" \
|
||||
-H "X-Llmgw-Knowledge: politikalar" \
|
||||
-d '{"model": "qwen2.5-coder-14b-instruct",
|
||||
"messages": [{"role": "user", "content": "Kaç gün yıllık iznim var?"}]}'
|
||||
```
|
||||
|
||||
```python
|
||||
client.chat.completions.create(
|
||||
model="qwen2.5-coder-14b-instruct",
|
||||
messages=[{"role": "user", "content": "Kaç gün yıllık iznim var?"}],
|
||||
extra_headers={"X-Llmgw-Knowledge": "politikalar"},
|
||||
)
|
||||
```
|
||||
|
||||
What the gateway does with such a request:
|
||||
|
||||
1. **It finds the collections.** It looks up the ones named, by name or ID,
|
||||
among those the key may read. A collection it may not read is answered
|
||||
404, the same as one that does not exist.
|
||||
2. **It embeds the question.** The last user message is embedded by the
|
||||
collection's model, on the caller's own account: their limits, budget and
|
||||
guardrails apply.
|
||||
3. **It searches.** The collections are searched by meaning and by words
|
||||
(below).
|
||||
4. **It adds the passages.** The best passages are numbered and added to the
|
||||
system message, with an instruction to answer from them, cite them as
|
||||
[1], [2], and say so when they do not hold the answer. The system message
|
||||
is created if the request has none. Passages that would not fit the
|
||||
model's context are left out, the weakest first.
|
||||
5. **It judges and sends the request.** The request, passages included, then
|
||||
goes through the guardrails and to the model as usual. A document
|
||||
carrying instructions aimed at its reader is judged like anything else
|
||||
the model reads.
|
||||
|
||||
On the portal's chat page, **Documents** picks a collection the person may
|
||||
read in the chosen project, the default, or none. Under each answer, the
|
||||
sources it was given are listed by number.
|
||||
|
||||
How it is shaped:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| `X-Llmgw-Knowledge` | Collections, comma separated; `none` for no passages, whatever the default. |
|
||||
| `X-Llmgw-Knowledge-K` | How many passages, 1 to 20; 5 by default. |
|
||||
| `X-Llmgw-Project` | For a personal key: the project it acts in. |
|
||||
| `x-llmgw-sources` (answer) | The passages given, as `document#chunk`, in their numbered order; empty when nothing matched. |
|
||||
|
||||
Behaviour to know about:
|
||||
|
||||
- **No passages matched.** The model is told that nothing in the documents
|
||||
answers the question, and asked to say so rather than answer from general
|
||||
knowledge.
|
||||
- **Other APIs.** Requests through `/v1/responses` and `/v1/messages`
|
||||
(Codex, Claude Code) work the same way, since they are answered as chat
|
||||
completions.
|
||||
- **Separate prefix caches.** On vLLM replicas, a request given passages
|
||||
carries a `cache_salt` derived from its organization. One organization's
|
||||
passages then never share cached blocks with another's, so how fast an
|
||||
answer starts says nothing about what another tenant asked.
|
||||
- **No answer cache.** Answers from knowledge bases are never served from
|
||||
the answer cache, because two keys of one project can read different
|
||||
collections.
|
||||
|
||||
### How a passage is chosen
|
||||
|
||||
Each passage scores the following, and the highest scores win:
|
||||
|
||||
- its similarity in meaning to the question (cosine, 0 to 1), plus
|
||||
- up to 0.3 for the share of the question's words it contains.
|
||||
|
||||
Words almost every passage has ("ne", "için", "the", "of") are not counted.
|
||||
The meaning does most of the work: "how much leave do fathers get" finds the
|
||||
paternity page, not the annual-leave page that shares more of its words.
|
||||
The word match helps with what meaning cannot carry, such as a form number,
|
||||
a product code or a name.
|
||||
|
||||
Metrics: `llmgw_kb_retrievals_total{result}` (found, nothing, refused) and
|
||||
`llmgw_kb_retrieval_seconds`.
|
||||
|
||||
## How it is kept apart
|
||||
|
||||
Passages live in PostgreSQL, partitioned by organization, with an index per
|
||||
collection. Every search and every write names the organization and the
|
||||
collections the caller may use. On top of that, row-level security on the
|
||||
table checks the same thing again, so a query that forgot its filter
|
||||
returns nothing rather than another tenant's passages.
|
||||
+27
-5
@@ -379,16 +379,38 @@ is what someone gets from a borrowed laptop, and the password is what they do no
|
||||
have — and it signs every other browser out, which is the point if they are
|
||||
changing it because something went wrong.
|
||||
|
||||
There is no self-service reset, because this gateway sends no email and a reset
|
||||
link with nowhere to send it is not a reset. When someone loses theirs:
|
||||
Someone who has forgotten theirs follows **Forgotten your password?** on the
|
||||
sign-in page. With mail set up in the cluster's settings, a link to set a new one
|
||||
is mailed to their address — the page says the same whether or not the address
|
||||
has an account, so nobody can find out who does from it:
|
||||
|
||||
```
|
||||
llmgw ctl config set mail '{smtp: smtp.example.com:587, from: llmgw@example.com,
|
||||
username: llmgw, password_file: /etc/llmgw/secrets/smtp, base_url: https://llm.example.com}'
|
||||
```
|
||||
|
||||
`base_url` is where the link points, and it is never taken from the request:
|
||||
otherwise a stranger could ask for someone's link from a page of their own. A
|
||||
link works once, for an hour, and a newer one replaces it.
|
||||
|
||||
Without mail, the page says to ask an operator, who makes the link and hands it
|
||||
over however suits:
|
||||
|
||||
```
|
||||
llmgw users reset-link ayse@example.com --store … # --base-url without mail.base_url
|
||||
```
|
||||
|
||||
Or sets a temporary password to give them, and has them change it from their
|
||||
profile:
|
||||
|
||||
```
|
||||
llmgw users reset-password usr-k4mq… --password "a temporary passphrase" --store …
|
||||
```
|
||||
|
||||
Tell them out of band and have them change it from their profile. Their sessions
|
||||
are closed — if they lost the password, the reason might be that someone else has
|
||||
it. Their API keys keep working; those are separate credentials.
|
||||
Either way their sessions are closed — if they lost the password, the reason
|
||||
might be that someone else has it. Their API keys keep working; those are
|
||||
separate credentials. People who sign in through single sign-on have no password
|
||||
here, and get no link.
|
||||
|
||||
## Trying a model in the browser
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ install:
|
||||
- your administrator account. **The password is shown once. Copy it now.**
|
||||
- a model mirror, which downloads model files once for the whole cluster;
|
||||
- this machine as a first node, and a small model (Qwen2.5 0.5B) on it;
|
||||
- guardrails on every request, with a judge model (Llama Guard 3 1B, two
|
||||
- guardrails on every request, with a judge model (Qwen3Guard-Gen 0.6B, two
|
||||
replicas) on CPUs — a request it cannot check is refused, not let through — and API keys that expire within 90 days. What is hardened, and how
|
||||
to loosen it: [install.md](install.md#2-the-control-plane).
|
||||
|
||||
|
||||
+114
-5
@@ -102,9 +102,11 @@ rather than repeated.
|
||||
| `format` | `string` | [GuardModel](#guardmodel) |
|
||||
| `categories` | `[]string` | [GuardModel](#guardmodel) |
|
||||
| `model` | `string` | [JudgePolicy](#judgepolicy) |
|
||||
| `models` | `[]string` | [JudgePolicy](#judgepolicy) |
|
||||
| `input` | `string` | [JudgePolicy](#judgepolicy) |
|
||||
| `output` | `string` | [JudgePolicy](#judgepolicy) |
|
||||
| `categories` | `[]string` | [JudgePolicy](#judgepolicy) |
|
||||
| `controversial` | `string` | [JudgePolicy](#judgepolicy) |
|
||||
| `latency_budget` | `time.Duration` | [JudgePolicy](#judgepolicy) |
|
||||
| `tokenize_timeout` | `time.Duration` | [KVIndex](#kvindex) |
|
||||
| `min_blocks` | `int` | [KVIndex](#kvindex) |
|
||||
@@ -203,6 +205,7 @@ rather than repeated.
|
||||
| `files` | `[]OriginFile` | [PreviousOrigin](#previousorigin) |
|
||||
| `name` | `string` | [Profile](#profile) |
|
||||
| `engine` | `string` | [Profile](#profile) |
|
||||
| `engine_version` | `string` | [Profile](#profile) |
|
||||
| `gpu_class` | `string` | [Profile](#profile) |
|
||||
| `tp` | `int` | [Profile](#profile) |
|
||||
| `pp` | `int` | [Profile](#profile) |
|
||||
@@ -288,6 +291,8 @@ rather than repeated.
|
||||
| `guard` | `guard.Config` | [Server](#server) |
|
||||
| `secrets` | `Secrets` | [Server](#server) |
|
||||
| `notify` | `notify.Config` | [Server](#server) |
|
||||
| `mail` | `mail.Config` | [Server](#server) |
|
||||
| `serving` | `ServingSettings` | [Server](#server) |
|
||||
| `firewall` | `FirewallSettings` | [Server](#server) |
|
||||
| `firewall_networks` | `[]string` | [Server](#server) |
|
||||
| `backup_recipients` | `[]string` | [Server](#server) |
|
||||
@@ -300,6 +305,7 @@ rather than repeated.
|
||||
| `facility` | `Facility` | [Server](#server) |
|
||||
| `federation` | `Federation` | [Server](#server) |
|
||||
| `endpoints` | `[]Endpoint` | [Server](#server) |
|
||||
| `stopped` | `bool` | [ServingSettings](#servingsettings) |
|
||||
| `bin` | `string` | [SimConfig](#simconfig) |
|
||||
| `python` | `string` | [TRTLLMConfig](#trtllmconfig) |
|
||||
| `store` | `string` | [Threads](#threads) |
|
||||
@@ -372,8 +378,14 @@ rather than repeated.
|
||||
| `llamacpp` | `LlamaCppConfig` | [engines.Config](#enginesconfig) |
|
||||
| `trtllm` | `TRTLLMConfig` | [engines.Config](#enginesconfig) |
|
||||
| `sim` | `SimConfig` | [engines.Config](#enginesconfig) |
|
||||
| `enabled` | `*bool` | [guard.Config](#guardconfig) |
|
||||
| `default` | `[]string` | [guard.Config](#guardconfig) |
|
||||
| `policies` | `map[string]Policy` | [guard.Config](#guardconfig) |
|
||||
| `smtp` | `string` | [mail.Config](#mailconfig) |
|
||||
| `from` | `string` | [mail.Config](#mailconfig) |
|
||||
| `username` | `string` | [mail.Config](#mailconfig) |
|
||||
| `password_file` | `string` | [mail.Config](#mailconfig) |
|
||||
| `base_url` | `string` | [mail.Config](#mailconfig) |
|
||||
| `webhooks` | `[]Webhook` | [notify.Config](#notifyconfig) |
|
||||
| `host` | `string` | [notify.Config](#notifyconfig) |
|
||||
| `work_dir` | `string` | [supervisor.Config](#supervisorconfig) |
|
||||
@@ -984,9 +996,11 @@ every node and control plane within seconds.
|
||||
|
||||
`string`
|
||||
|
||||
Format is llamaguard3 (Meta's Llama Guard 3: "safe", or "unsafe" and
|
||||
the categories), shieldgemma (Google's ShieldGemma: Yes or No), or
|
||||
json ({"safe": false, "categories": ["..."]}).
|
||||
Format is qwen3guard (Qwen3Guard-Gen: "Safety: Safe", "Unsafe" or
|
||||
"Controversial", and the categories), llamaguard3 (Meta's Llama Guard
|
||||
3: "safe", or "unsafe" and the categories), shieldgemma (Google's
|
||||
ShieldGemma: Yes or No), or json ({"safe": false, "categories":
|
||||
["..."]}).
|
||||
|
||||
### `categories`
|
||||
|
||||
@@ -1003,6 +1017,15 @@ sentence — and the names reported for llamaguard3's S1, S2, ….
|
||||
|
||||
Model is a catalog model with role: guard.
|
||||
|
||||
### `models`
|
||||
|
||||
`[]string`
|
||||
|
||||
Models are judges asked in turn, instead of Model: the first that
|
||||
answers gives the verdict. A judge on a GPU first and one on CPUs
|
||||
after it is the GPU's speed while it is there, and the CPUs' when it
|
||||
is not — busy, failed, or the node down.
|
||||
|
||||
### `input`
|
||||
|
||||
`string`
|
||||
@@ -1021,12 +1044,19 @@ Empty does not ask the judge about that direction.
|
||||
Categories, when set, are the only ones that count: Llama Guard's
|
||||
codes (S1, S10) or the names the model reports.
|
||||
|
||||
### `controversial`
|
||||
|
||||
`string`
|
||||
|
||||
Controversial is what Qwen3Guard's third verdict counts as: safe
|
||||
(the default) or unsafe.
|
||||
|
||||
### `latency_budget`
|
||||
|
||||
`time.Duration`
|
||||
|
||||
LatencyBudget is how long the judge is waited for before the policy's
|
||||
fail applies. Zero is 250ms.
|
||||
LatencyBudget is how long each judge is waited for before the next is
|
||||
asked, or, after the last, the policy's fail applies. Zero is 250ms.
|
||||
|
||||
## KVIndex
|
||||
|
||||
@@ -1755,6 +1785,14 @@ before affinity is abandoned.
|
||||
|
||||
`string`
|
||||
|
||||
### `engine_version`
|
||||
|
||||
`string`
|
||||
|
||||
EngineVersion restricts the profile to nodes whose installed engine's
|
||||
version starts with it: "0.11" takes vLLM 0.11.0 and 0.11.1, not
|
||||
0.10.2. Empty takes whatever version the node has.
|
||||
|
||||
### `gpu_class`
|
||||
|
||||
`string`
|
||||
@@ -2456,6 +2494,24 @@ Prometheus and Alertmanager in between. Empty sends nothing.
|
||||
|
||||
Takes a [notify.Config](#notifyconfig) block.
|
||||
|
||||
### `mail`
|
||||
|
||||
`mail.Config`
|
||||
|
||||
Mail is where the gateway's few emails go out from: a link to set a
|
||||
forgotten password. Unset, none is sent and people ask an operator.
|
||||
|
||||
Takes a [mail.Config](#mailconfig) block.
|
||||
|
||||
### `serving`
|
||||
|
||||
`ServingSettings`
|
||||
|
||||
Serving stops and starts the whole cluster's service (llmgw stop, llmgw
|
||||
start).
|
||||
|
||||
Takes a [ServingSettings](#servingsettings) block.
|
||||
|
||||
### `firewall`
|
||||
|
||||
`FirewallSettings`
|
||||
@@ -2554,6 +2610,18 @@ Endpoints is the phase-1 static replica list. It is only consulted when
|
||||
no model declares a profile, which is what makes a router-only smoke test
|
||||
against sim-engines possible without a control plane.
|
||||
|
||||
## ServingSettings
|
||||
|
||||
### `stopped`
|
||||
|
||||
`bool`
|
||||
|
||||
Stopped takes every model down and answers every request for one with
|
||||
503 (system_stopped), leaving the control planes, agents and mirror
|
||||
running and every model's definition as it is: llmgw stop sets it,
|
||||
llmgw start clears it, and the models come back as their catalog
|
||||
entries ask. In force on every control plane within seconds.
|
||||
|
||||
## SimConfig
|
||||
|
||||
### `bin`
|
||||
@@ -3067,6 +3135,13 @@ Takes a [SimConfig](#simconfig) block.
|
||||
|
||||
## guard.Config
|
||||
|
||||
### `enabled`
|
||||
|
||||
`*bool`
|
||||
|
||||
Enabled turns every policy on or off at once: false checks nothing,
|
||||
whatever is defined or attached. Unset is on.
|
||||
|
||||
### `default`
|
||||
|
||||
`[]string`
|
||||
@@ -3075,6 +3150,40 @@ Takes a [SimConfig](#simconfig) block.
|
||||
|
||||
`map[string]Policy`
|
||||
|
||||
## mail.Config
|
||||
|
||||
### `smtp`
|
||||
|
||||
`string`
|
||||
|
||||
SMTP is the relay, host:port: 587 with STARTTLS, 465 with TLS from the
|
||||
start, 25 as the relay allows. Empty sends nothing.
|
||||
|
||||
### `from`
|
||||
|
||||
`string`
|
||||
|
||||
From is the sender, llmgw@example.com or "llmgw <llmgw@example.com>".
|
||||
|
||||
### `username`
|
||||
|
||||
`string`
|
||||
|
||||
Username and PasswordFile log in to the relay, when it wants that.
|
||||
The password is a secret reference: a file, env:NAME or vault:path.
|
||||
|
||||
### `password_file`
|
||||
|
||||
`string`
|
||||
|
||||
### `base_url`
|
||||
|
||||
`string`
|
||||
|
||||
BaseURL is the gateway's public address, which links in mail point
|
||||
at: https://llm.example.com. Never taken from a request, whose Host a
|
||||
sender chooses and could point a reset link at their own site.
|
||||
|
||||
## notify.Config
|
||||
|
||||
### `webhooks`
|
||||
|
||||
@@ -18,6 +18,7 @@ require (
|
||||
github.com/knadh/koanf/providers/env/v2 v2.0.1
|
||||
github.com/knadh/koanf/providers/file v1.2.1
|
||||
github.com/knadh/koanf/v2 v2.3.6
|
||||
github.com/ledongthuc/pdf v0.0.0-20260907135840-6c8c28e0e8a0
|
||||
github.com/mattn/go-isatty v0.0.24
|
||||
github.com/prometheus/client_golang v1.24.1
|
||||
github.com/prometheus/client_model v0.6.3
|
||||
@@ -34,6 +35,7 @@ require (
|
||||
golang.org/x/net v0.59.0
|
||||
golang.org/x/oauth2 v0.37.0
|
||||
golang.org/x/sync v0.23.0
|
||||
golang.org/x/term v0.46.0
|
||||
golang.org/x/text v0.42.0
|
||||
google.golang.org/protobuf v1.36.12
|
||||
modernc.org/sqlite v1.59.0
|
||||
|
||||
@@ -95,6 +95,8 @@ github.com/knadh/koanf/v2 v2.3.6 h1:JoQPSJmvS4aP0xNc8xMDr5tcrkSEInL23/Il7pITAKo=
|
||||
github.com/knadh/koanf/v2 v2.3.6/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/ledongthuc/pdf v0.0.0-20260907135840-6c8c28e0e8a0 h1:7Q+xNAZFmnfYOMweHN3c/PDFUKKfY1pVJ26K++QvVfU=
|
||||
github.com/ledongthuc/pdf v0.0.0-20260907135840-6c8c28e0e8a0/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||
|
||||
@@ -448,3 +448,40 @@ func TestAHashThatCannotGetATurnSaysTheServerIsBusy(t *testing.T) {
|
||||
t.Errorf("a hash with no turn: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAForgottenPasswordIsResetOnceThroughALink(t *testing.T) {
|
||||
svc, st := newService(t)
|
||||
ctx := context.Background()
|
||||
must(t, svc.SignUp(ctx, "ayse@example.com", "A", goodPassword))
|
||||
u, _ := st.UserByEmail(ctx, "ayse@example.com")
|
||||
|
||||
// Waiting for approval: no link, said the same as for nobody.
|
||||
if _, _, err := svc.RequestReset(ctx, st, "ayse@example.com"); !errors.Is(err, ErrNoReset) {
|
||||
t.Fatalf("a pending account got a link: %v", err)
|
||||
}
|
||||
if _, _, err := svc.RequestReset(ctx, st, "nobody@example.com"); !errors.Is(err, ErrNoReset) {
|
||||
t.Fatalf("an unknown address: %v", err)
|
||||
}
|
||||
must(t, st.ApproveUser(ctx, u.ID, "proj", "ilker", time.Now()))
|
||||
session, _, err := svc.SignIn(ctx, "ayse@example.com", goodPassword)
|
||||
must(t, err)
|
||||
|
||||
token, got, err := svc.RequestReset(ctx, st, "AYSE@example.com")
|
||||
if err != nil || token == "" || got.ID != u.ID {
|
||||
t.Fatalf("an active account: %q %+v %v", token, got, err)
|
||||
}
|
||||
if err := svc.ResetPassword(ctx, st, st, token, "short"); !errors.Is(err, ErrWeakPassword) {
|
||||
t.Fatalf("a weak password: %v", err)
|
||||
}
|
||||
next := goodPassword + "-new"
|
||||
must(t, svc.ResetPassword(ctx, st, st, token, next))
|
||||
if _, _, err := svc.Resolve(ctx, session); err == nil {
|
||||
t.Error("a session from before the reset still works")
|
||||
}
|
||||
if _, _, err := svc.SignIn(ctx, "ayse@example.com", next); err != nil {
|
||||
t.Errorf("the new password does not sign in: %v", err)
|
||||
}
|
||||
if err := svc.ResetPassword(ctx, st, st, token, next+"2"); !errors.Is(err, ErrNoReset) {
|
||||
t.Errorf("a link used twice: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -373,3 +373,63 @@ func (s *Service) OpenSession(ctx context.Context, userID string) (token, csrf s
|
||||
}
|
||||
return token, csrf, nil
|
||||
}
|
||||
|
||||
// ResetLinkTTL is how long a password reset link works.
|
||||
const ResetLinkTTL = time.Hour
|
||||
|
||||
// ResetStore keeps password reset links.
|
||||
type ResetStore interface {
|
||||
UserByEmail(ctx context.Context, email string) (store.User, error)
|
||||
CreatePasswordReset(ctx context.Context, userID string, ttl time.Duration) (string, error)
|
||||
PasswordResetUser(ctx context.Context, token string) (string, error)
|
||||
UsePasswordReset(ctx context.Context, token string) (string, error)
|
||||
}
|
||||
|
||||
// ErrNoReset is an address that gets no reset link: none has an account, the
|
||||
// account signs in through single sign-on, or it is not active. Never told to
|
||||
// whoever asked, who learns the same either way.
|
||||
var ErrNoReset = errors.New("no reset link for that address")
|
||||
|
||||
// RequestReset mints a reset link token for the person an address belongs
|
||||
// to, when they sign in with a password and are active. A few a quarter of
|
||||
// an hour per address: each sends a mail.
|
||||
func (s *Service) RequestReset(ctx context.Context, rs ResetStore, email string) (string, store.User, error) {
|
||||
email = store.NormalizeEmail(email)
|
||||
if !validEmail(email) {
|
||||
return "", store.User{}, ErrNoReset
|
||||
}
|
||||
if !s.allowAttempt("reset:"+email, 3) {
|
||||
return "", store.User{}, ErrTooManyTries
|
||||
}
|
||||
u, err := rs.UserByEmail(ctx, email)
|
||||
if err != nil || u.PasswordHash == "" || u.State != store.UserActive {
|
||||
return "", store.User{}, ErrNoReset
|
||||
}
|
||||
token, err := rs.CreatePasswordReset(ctx, u.ID, ResetLinkTTL)
|
||||
if err != nil {
|
||||
return "", store.User{}, err
|
||||
}
|
||||
s.log.Info("a password reset link was made", "user", u.ID)
|
||||
return token, u, nil
|
||||
}
|
||||
|
||||
// ResetPassword sets a password from a reset link, which is spent, and signs
|
||||
// the person out everywhere: if they lost it, someone else may have it.
|
||||
func (s *Service) ResetPassword(ctx context.Context, rs ResetStore, pw PasswordChanger, token, next string) error {
|
||||
hash, err := HashPassword(next)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
userID, err := rs.UsePasswordReset(ctx, token)
|
||||
if err != nil {
|
||||
return ErrNoReset
|
||||
}
|
||||
if err := pw.SetPassword(ctx, userID, hash); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := pw.DeleteUserSessions(ctx, userID); err != nil {
|
||||
return err
|
||||
}
|
||||
s.log.Info("a password was set from a reset link", "user", userID)
|
||||
return nil
|
||||
}
|
||||
+114
-3
@@ -371,6 +371,12 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) {
|
||||
_ = a.sup.Stop(ctx, o.ShardID)
|
||||
return
|
||||
}
|
||||
// What it computes on, for one started before the agent kept that.
|
||||
if a.sup.ComputeOf(o.ShardID) == nil {
|
||||
if c, ok := a.gpuCompute(ctx, o.Spec.GPUIndices, sh.Status().PID); ok {
|
||||
a.sup.SetCompute(o.ShardID, c)
|
||||
}
|
||||
}
|
||||
a.log.Info("adopted engine is serving",
|
||||
"shard", o.ShardID, "replica", o.ReplicaID)
|
||||
}
|
||||
@@ -411,6 +417,17 @@ func (a *Agent) weights() *weights.Fetcher {
|
||||
}
|
||||
|
||||
// runningShards describes what this agent is supervising, for registration.
|
||||
// unready are the shards a registration reported as not yet ready.
|
||||
func unready(running []*llmgwv1.RunningShard) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, r := range running {
|
||||
if r.GetState() != agentsvc.StateToWire(supervisor.Ready) {
|
||||
out[r.GetShardId()] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *Agent) runningShards() []*llmgwv1.RunningShard {
|
||||
var out []*llmgwv1.RunningShard
|
||||
for _, st := range a.sup.Statuses() {
|
||||
@@ -436,6 +453,7 @@ func (a *Agent) runningShards() []*llmgwv1.RunningShard {
|
||||
Pool: spec.Pool,
|
||||
Fabric: spec.Fabric,
|
||||
ProfileFp: spec.ProfileFP,
|
||||
Compute: computeToWire(a.sup.ComputeOf(st.ID)),
|
||||
LeaderAddr: a.cfg.MgmtAddr,
|
||||
LeaderPort: int32(spec.LeaderPort),
|
||||
})
|
||||
@@ -564,8 +582,9 @@ func (a *Agent) session(ctx context.Context, server string) error {
|
||||
a.sendLoop(ctx, stream, out)
|
||||
}()
|
||||
defer wg.Wait()
|
||||
go a.watchExits(ctx, out)
|
||||
go a.watchExits(ctx, out, unready(reg.RunningShards))
|
||||
go a.forwardKV(ctx, out)
|
||||
go a.reportCompute(ctx, out)
|
||||
|
||||
for {
|
||||
msg, err := stream.Receive()
|
||||
@@ -861,6 +880,14 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
|
||||
a.setControlPlanes(m.ControlPlanes)
|
||||
case *llmgwv1.ControlMessage_Firewall:
|
||||
go a.applyFirewall(m.Firewall)
|
||||
case *llmgwv1.ControlMessage_RestartAgent:
|
||||
go func() {
|
||||
a.log.Info("restarting, as the control plane asks; the engines keep serving",
|
||||
"reason", m.RestartAgent.GetReason())
|
||||
// A moment for the log line to leave before the process does.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
exitForUpgrade()
|
||||
}()
|
||||
case *llmgwv1.ControlMessage_Logs:
|
||||
reply := a.logsReply(m.Logs)
|
||||
// Off the receive loop: a full send queue must not stop commands being
|
||||
@@ -1130,6 +1157,14 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
} else {
|
||||
err = a.sup.WaitAlive(ctx, sh)
|
||||
}
|
||||
// Where it put the model, by its own account: said with the readiness,
|
||||
// and a GPU placement it computes on the CPU refused even where
|
||||
// nvidia-smi cannot be asked.
|
||||
compute, said, cerr := engineCompute(eng, spec.GPUIndices, sh.Logs(512))
|
||||
if err == nil && cerr != nil {
|
||||
err = cerr
|
||||
_ = a.sup.Stop(ctx, shardID)
|
||||
}
|
||||
if err == nil {
|
||||
err = a.onItsGPUs(ctx, eng, spec, sh.Status().PID)
|
||||
if err != nil {
|
||||
@@ -1142,7 +1177,19 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
report(supervisor.Failed, err.Error(), sh.Logs(20))
|
||||
return
|
||||
}
|
||||
report(supervisor.Ready, "", nil)
|
||||
// The engine's own account when it gave one, with its layers; else what
|
||||
// nvidia-smi sees the shard holding.
|
||||
if !said {
|
||||
compute, said = a.gpuCompute(ctx, spec.GPUIndices, sh.Status().PID)
|
||||
}
|
||||
if said {
|
||||
// Kept with the shard, so an agent restarted, or a control plane
|
||||
// taking over, still learns it.
|
||||
a.sup.SetCompute(shardID, compute)
|
||||
send(&llmgwv1.ShardStatus{State: agentsvc.StateToWire(supervisor.Ready), Compute: computeToWire(&compute)})
|
||||
} else {
|
||||
report(supervisor.Ready, "", nil)
|
||||
}
|
||||
// Its dying later is reported by the session's watchExits.
|
||||
}
|
||||
|
||||
@@ -1155,7 +1202,14 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
// reported the engine dying; and an adopted engine had no watcher at all —
|
||||
// on node06 one was killed and its replica stayed "ready" for the control
|
||||
// plane, routed to and failing, indefinitely.
|
||||
func (a *Agent) watchExits(ctx context.Context, out chan<- *llmgwv1.AgentMessage) {
|
||||
//
|
||||
// It also reports the shards the registration said were not ready yet once
|
||||
// they are. An adopted engine is health-checked after the agent registers,
|
||||
// and a shard started on the stream before is reported on that stream; so
|
||||
// when the registration came first nothing ever told the control plane, and
|
||||
// the replica stayed "starting" while it served — on node05, after a
|
||||
// control plane restart, the only replica of a model.
|
||||
func (a *Agent) watchExits(ctx context.Context, out chan<- *llmgwv1.AgentMessage, pending map[string]bool) {
|
||||
t := time.NewTicker(time.Second)
|
||||
defer t.Stop()
|
||||
ready := map[string]bool{}
|
||||
@@ -1164,6 +1218,24 @@ func (a *Agent) watchExits(ctx context.Context, out chan<- *llmgwv1.AgentMessage
|
||||
switch st.State {
|
||||
case supervisor.Ready:
|
||||
ready[st.ID] = true
|
||||
if !pending[st.ID] {
|
||||
continue
|
||||
}
|
||||
delete(pending, st.ID)
|
||||
sh, err := a.sup.Get(st.ID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||
NodeId: a.cfg.NodeID, ReplicaId: sh.Spec.ReplicaID, ShardId: st.ID,
|
||||
State: agentsvc.StateToWire(st.State),
|
||||
Compute: computeToWire(a.sup.ComputeOf(st.ID)),
|
||||
}}}
|
||||
select {
|
||||
case out <- msg:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
case supervisor.Failed:
|
||||
if !ready[st.ID] {
|
||||
continue // never ready here: its start reported it
|
||||
@@ -1373,3 +1445,42 @@ func ResolveManagement(c Config) (addr, iface string) {
|
||||
}
|
||||
return addr, iface
|
||||
}
|
||||
|
||||
// reportCompute tells the control plane, once a session, what each running
|
||||
// shard computes on as soon as that is known: an adopted engine's is found
|
||||
// after registration, which carried what was known then.
|
||||
func (a *Agent) reportCompute(ctx context.Context, out chan<- *llmgwv1.AgentMessage) {
|
||||
sent := map[string]bool{}
|
||||
t := time.NewTicker(5 * time.Second)
|
||||
defer t.Stop()
|
||||
for range 60 {
|
||||
for _, st := range a.sup.Statuses() {
|
||||
if st.State.Terminal() || sent[st.ID] {
|
||||
continue
|
||||
}
|
||||
c := a.sup.ComputeOf(st.ID)
|
||||
if c == nil {
|
||||
continue
|
||||
}
|
||||
sh, err := a.sup.Get(st.ID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// No state: a report of compute alone, not a transition.
|
||||
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||
NodeId: a.cfg.NodeID, ReplicaId: sh.Spec.ReplicaID, ShardId: st.ID, Compute: computeToWire(c),
|
||||
}}}
|
||||
select {
|
||||
case out <- msg:
|
||||
sent[st.ID] = true
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -37,11 +37,12 @@ import (
|
||||
type statusRecorder struct {
|
||||
ch chan status
|
||||
|
||||
mu sync.Mutex
|
||||
calls []adoptCall
|
||||
adopted []reconciler.AdoptedShard
|
||||
unclaim []string // shard ids to report back as unclaimed
|
||||
fetches []fetchReport
|
||||
mu sync.Mutex
|
||||
calls []adoptCall
|
||||
adopted []reconciler.AdoptedShard
|
||||
unclaim []string // shard ids to report back as unclaimed
|
||||
fetches []fetchReport
|
||||
computes []reconciler.Compute
|
||||
}
|
||||
|
||||
type status struct {
|
||||
@@ -61,6 +62,12 @@ func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor.
|
||||
|
||||
// fetches records download progress, so a test can assert that a fetch is
|
||||
// visible from the control plane rather than only in the node's log.
|
||||
func (r *statusRecorder) OnShardCompute(replicaID, shardID string, c reconciler.Compute) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.computes = append(r.computes, c)
|
||||
}
|
||||
|
||||
func (r *statusRecorder) OnFetchProgress(replicaID, shardID, source string, done, total int64) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Compute is where an engine runs its model, as it said while loading it.
|
||||
type Compute struct {
|
||||
// Backend is CUDA, Vulkan, Metal, ROCm or CPU.
|
||||
Backend string
|
||||
// Devices are those it named: "CUDA0: Quadro RTX 6000 (7.5)".
|
||||
Devices []string
|
||||
// Offloaded of Total layers went to the devices; 0 and 0 when it did
|
||||
// not say.
|
||||
Offloaded, Total int
|
||||
// NoGPU is the engine saying it found no device it could use.
|
||||
NoGPU bool
|
||||
}
|
||||
|
||||
var (
|
||||
offloadedRE = regexp.MustCompile(`offloaded (\d+)/(\d+) layers to GPU`)
|
||||
usingRE = regexp.MustCompile(`using device ([A-Za-z]+\d+) \(([^)]*)\)`)
|
||||
ccRE = regexp.MustCompile(`Device (\d+): ([^,]+), compute capability ([0-9.]+)`)
|
||||
noDevicesRE = regexp.MustCompile(`found 0 (CUDA|Vulkan|ROCm) devices|no usable GPU found`)
|
||||
)
|
||||
|
||||
// ReadLlamaCompute reads llama.cpp's start-up output for where it put the
|
||||
// model, reporting false when the output says nothing of it.
|
||||
func ReadLlamaCompute(lines []string) (Compute, bool) {
|
||||
var c Compute
|
||||
known := false
|
||||
cc := map[string]string{} // device index → compute capability
|
||||
for _, l := range lines {
|
||||
if m := ccRE.FindStringSubmatch(l); m != nil {
|
||||
cc[m[1]] = m[3]
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, l := range lines {
|
||||
if m := offloadedRE.FindStringSubmatch(l); m != nil {
|
||||
c.Offloaded, _ = strconv.Atoi(m[1])
|
||||
c.Total, _ = strconv.Atoi(m[2])
|
||||
known = true
|
||||
}
|
||||
if m := usingRE.FindStringSubmatch(l); m != nil && !seen[m[1]] {
|
||||
seen[m[1]] = true
|
||||
d := m[1] + ": " + strings.TrimSpace(m[2])
|
||||
idx := strings.TrimLeft(m[1], "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz")
|
||||
if v, ok := cc[idx]; ok && strings.HasPrefix(m[1], "CUDA") {
|
||||
d += " (" + v + ")"
|
||||
}
|
||||
c.Devices = append(c.Devices, d)
|
||||
if c.Backend == "" {
|
||||
c.Backend = strings.TrimRight(m[1], "0123456789")
|
||||
}
|
||||
known = true
|
||||
}
|
||||
if noDevicesRE.MatchString(l) {
|
||||
c.NoGPU, known = true, true
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case c.OnTheCPU():
|
||||
c.Backend = "CPU"
|
||||
case c.Backend == "" && c.Offloaded > 0:
|
||||
c.Backend = "GPU"
|
||||
case c.Backend == "":
|
||||
c.Backend = "CPU"
|
||||
}
|
||||
return c, known
|
||||
}
|
||||
|
||||
// OnTheCPU is whether a model the engine was given GPUs for is computed on
|
||||
// the CPU, by its own account: no device found, or no layer offloaded.
|
||||
func (c Compute) OnTheCPU() bool {
|
||||
return c.NoGPU || (c.Total > 0 && c.Offloaded == 0)
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLlamaCppSaysWhereItPutTheModel(t *testing.T) {
|
||||
gpu := strings.Split(`ggml_cuda_init: found 1 CUDA devices:
|
||||
Device 0: Quadro RTX 6000, compute capability 7.5, VMM: yes
|
||||
load_backend: loaded CUDA backend from /opt/llmgw/engines/llamacpp/b10991/libggml-cuda.so
|
||||
llama_model_load_from_file_impl: using device CUDA0 (Quadro RTX 6000) - 23800 MiB free
|
||||
load_tensors: offloading 48 repeating layers to GPU
|
||||
load_tensors: offloaded 49/49 layers to GPU`, "\n")
|
||||
c, ok := ReadLlamaCompute(gpu)
|
||||
if !ok || c.Backend != "CUDA" || c.Offloaded != 49 || c.Total != 49 || c.OnTheCPU() ||
|
||||
len(c.Devices) != 1 || c.Devices[0] != "CUDA0: Quadro RTX 6000 (7.5)" {
|
||||
t.Errorf("on the GPU: %+v %v", c, ok)
|
||||
}
|
||||
|
||||
cpu := strings.Split(`warning: no usable GPU found, --gpu-layers option will be ignored
|
||||
warning: one possible reason is that llama.cpp was compiled without GPU support
|
||||
load_tensors: offloaded 0/49 layers to GPU`, "\n")
|
||||
c, ok = ReadLlamaCompute(cpu)
|
||||
if !ok || c.Backend != "CPU" || !c.OnTheCPU() {
|
||||
t.Errorf("fell back to the CPU: %+v %v", c, ok)
|
||||
}
|
||||
|
||||
if _, ok := ReadLlamaCompute([]string{"main: server is listening on http://127.0.0.1:20001"}); ok {
|
||||
t.Error("output that says nothing was read as saying something")
|
||||
}
|
||||
}
|
||||
@@ -114,6 +114,10 @@ type ShardSpec struct {
|
||||
// lets the control plane tell, after its own restart, whether a shard that
|
||||
// is still running matches the profile the catalog holds now.
|
||||
ProfileFP string
|
||||
|
||||
// Compute is where the engine said it put the model as it became ready,
|
||||
// kept with the shard's state so an agent restarted later still knows.
|
||||
Compute *Compute `json:",omitempty"`
|
||||
}
|
||||
|
||||
// Host is where the engine listens.
|
||||
|
||||
@@ -277,6 +277,9 @@ func (l *LlamaCpp) bin() string {
|
||||
return "llama-server"
|
||||
}
|
||||
|
||||
// Binary is the llama-server this engine runs.
|
||||
func (l *LlamaCpp) Binary() string { return l.bin() }
|
||||
|
||||
// FallsBackToCPU says that llama.cpp keeps running when its GPU backend does
|
||||
// not load: --n-gpu-layers asks for layers on a GPU it then has none of, and
|
||||
// the server comes up and answers on the CPU, at a tenth of the speed, with no
|
||||
|
||||
@@ -4,10 +4,12 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
)
|
||||
@@ -153,3 +155,70 @@ func joinIndices(is []int) string {
|
||||
}
|
||||
return strings.Join(s, ", ")
|
||||
}
|
||||
|
||||
// engineCompute is where an engine said it put its model, from its output
|
||||
// so far — llama.cpp's, which says — and false for one that said nothing. A
|
||||
// shard given GPUs that its engine says it computes on the CPU is an error,
|
||||
// found this way even where nvidia-smi cannot be asked.
|
||||
func engineCompute(eng engines.Engine, gpus []int, lines []string) (engines.Compute, bool, error) {
|
||||
if _, ok := eng.(engines.CPUFallback); !ok {
|
||||
return engines.Compute{}, false, nil
|
||||
}
|
||||
read, ok := engines.ReadLlamaCompute(lines)
|
||||
if !ok {
|
||||
return engines.Compute{}, false, nil
|
||||
}
|
||||
if len(gpus) > 0 && read.OnTheCPU() {
|
||||
why := "its GPU backend saw no device it could use"
|
||||
if read.Total > 0 && !read.NoGPU {
|
||||
why = fmt.Sprintf("it offloaded %d of %d layers", read.Offloaded, read.Total)
|
||||
}
|
||||
return read, true, fmt.Errorf("placed on GPU %v but computing on the CPU: %s, by llama.cpp's own account "+
|
||||
"(llmgw ctl engine-check on this node says why)", gpus, why)
|
||||
}
|
||||
return read, true, nil
|
||||
}
|
||||
|
||||
// computeToWire is a compute as it travels to the control plane.
|
||||
func computeToWire(c *engines.Compute) *llmgwv1.Compute {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
return &llmgwv1.Compute{Backend: c.Backend, Devices: c.Devices,
|
||||
LayersOffloaded: int32(c.Offloaded), LayersTotal: int32(c.Total)} //nolint:gosec // layer counts
|
||||
}
|
||||
|
||||
// gpuCompute is where a shard computes as nvidia-smi sees it: the CPU for a
|
||||
// shard given no GPU, else the GPUs whose memory its processes hold, and how
|
||||
// much. False when nvidia-smi cannot be asked. Engine-agnostic, and needing
|
||||
// no log line: llama.cpp says where its layers went only at -lv 4.
|
||||
func (a *Agent) gpuCompute(ctx context.Context, gpus []int, pid int) (engines.Compute, bool) {
|
||||
if len(gpus) == 0 {
|
||||
return engines.Compute{Backend: "CPU"}, true
|
||||
}
|
||||
mem, err := gpuMemory(ctx, a.cfg.Discovery.NvidiaSMI)
|
||||
if err != nil || mem == nil || pid <= 0 {
|
||||
return engines.Compute{}, false
|
||||
}
|
||||
group := processGroup(pid)
|
||||
c := engines.Compute{Backend: "CUDA"}
|
||||
for _, g := range mem {
|
||||
if !slices.Contains(gpus, g.Index) {
|
||||
continue
|
||||
}
|
||||
used := 0
|
||||
for _, p := range g.Procs {
|
||||
if holds([]int{p.PID}, pid, group) {
|
||||
used += p.UsedMB
|
||||
}
|
||||
}
|
||||
if used > 0 {
|
||||
c.Devices = append(c.Devices, fmt.Sprintf("GPU %d: %.1f GB", g.Index, float64(used)/1024))
|
||||
}
|
||||
}
|
||||
if len(c.Devices) == 0 {
|
||||
// Placed on GPUs and holding none of their memory.
|
||||
c.Backend = "CPU"
|
||||
}
|
||||
return c, true
|
||||
}
|
||||
@@ -122,3 +122,55 @@ func TestForeignGPUMemoryLeavesOutTheAgentsOwnShards(t *testing.T) {
|
||||
t.Errorf("no reading came back as %v", foreign)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLlamaCppsOwnWordOnTheCPUIsEnoughToRefuse(t *testing.T) {
|
||||
eng := fallbackEngine{}
|
||||
// Without nvidia-smi to ask, the engine's own output says it.
|
||||
c, said, err := engineCompute(eng, []int{0}, []string{
|
||||
"warning: no usable GPU found, --gpu-layers option will be ignored",
|
||||
"load_tensors: offloaded 0/49 layers to GPU"})
|
||||
if err == nil || !said || !strings.Contains(err.Error(), "computing on the CPU") || c.Backend != "CPU" {
|
||||
t.Fatalf("a GPU shard llama.cpp put on the CPU: %v %+v", err, c)
|
||||
}
|
||||
c, said, err = engineCompute(eng, []int{0}, []string{
|
||||
"llama_model_load_from_file_impl: using device CUDA0 (Quadro RTX 6000) - 23800 MiB free",
|
||||
"load_tensors: offloaded 49/49 layers to GPU"})
|
||||
if err != nil || !said || c.Backend != "CUDA" || c.Offloaded != 49 || computeToWire(&c).GetLayersOffloaded() != 49 {
|
||||
t.Errorf("on its GPU: %v %+v", err, c)
|
||||
}
|
||||
// A CPU shard is not refused for being on the CPU, and says so.
|
||||
if c, _, err := engineCompute(eng, nil, []string{"load_tensors: offloaded 0/49 layers to GPU"}); err != nil || c.Backend != "CPU" {
|
||||
t.Errorf("a CPU shard: %v %+v", err, c)
|
||||
}
|
||||
// An engine that said nothing says nothing.
|
||||
if _, said, err := engineCompute(eng, []int{0}, []string{"listening"}); err != nil || said {
|
||||
t.Errorf("nothing said: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNvidiaSMISaysWhereAShardComputes(t *testing.T) {
|
||||
oldMem, oldGroup := gpuMemory, processGroup
|
||||
t.Cleanup(func() { gpuMemory, processGroup = oldMem, oldGroup })
|
||||
processGroup = func(pid int) int { return pid }
|
||||
gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) {
|
||||
return []discovery.GPUMemory{
|
||||
{Index: 0, UsedMB: 20000, Procs: []discovery.GPUProcess{{PID: 700, UsedMB: 2048}, {PID: 900, UsedMB: 17000}}},
|
||||
{Index: 1, UsedMB: 0},
|
||||
}, nil
|
||||
}
|
||||
a := &Agent{log: slog.Default()}
|
||||
if c, ok := a.gpuCompute(context.Background(), []int{0}, 700); !ok || c.Backend != "CUDA" ||
|
||||
len(c.Devices) != 1 || c.Devices[0] != "GPU 0: 2.0 GB" {
|
||||
t.Errorf("on GPU 0: %+v %v", c, ok)
|
||||
}
|
||||
if c, ok := a.gpuCompute(context.Background(), []int{1}, 700); !ok || c.Backend != "CPU" {
|
||||
t.Errorf("placed on GPU 1, holding none of it: %+v", c)
|
||||
}
|
||||
if c, ok := a.gpuCompute(context.Background(), nil, 700); !ok || c.Backend != "CPU" {
|
||||
t.Errorf("a CPU shard: %+v", c)
|
||||
}
|
||||
gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) { return nil, nil }
|
||||
if _, ok := a.gpuCompute(context.Background(), []int{0}, 700); ok {
|
||||
t.Error("no nvidia-smi read as an answer")
|
||||
}
|
||||
}
|
||||
@@ -113,7 +113,10 @@ type Shard struct {
|
||||
exitCode int
|
||||
|
||||
logs *ringBuffer
|
||||
cmd *exec.Cmd
|
||||
// cmdline is an adopted engine's command line, from its state file: it
|
||||
// has no cmd of this agent's to read it from.
|
||||
cmdline []string
|
||||
cmd *exec.Cmd
|
||||
// sidecars are the shard's supporting processes, torn down with it.
|
||||
sidecars []*exec.Cmd
|
||||
cancel context.CancelFunc
|
||||
@@ -644,6 +647,32 @@ func (s *Supervisor) Get(id string) (*Shard, error) {
|
||||
return sh, nil
|
||||
}
|
||||
|
||||
// SetCompute records where a shard's engine said it put the model, with the
|
||||
// shard's state on disk.
|
||||
func (s *Supervisor) SetCompute(id string, c engines.Compute) {
|
||||
s.mu.Lock()
|
||||
sh, ok := s.shards[id]
|
||||
if ok {
|
||||
sh.Spec.Compute = &c
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if ok && sh.pid > 0 {
|
||||
_ = s.writeState(sh)
|
||||
}
|
||||
}
|
||||
|
||||
// ComputeOf is where a shard's engine was found to compute, nil when not
|
||||
// yet known.
|
||||
func (s *Supervisor) ComputeOf(id string) *engines.Compute {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if sh, ok := s.shards[id]; ok && sh.Spec.Compute != nil {
|
||||
c := *sh.Spec.Compute
|
||||
return &c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove forgets a terminated shard.
|
||||
func (s *Supervisor) Remove(id string) {
|
||||
s.mu.Lock()
|
||||
@@ -703,11 +732,17 @@ type shardState struct {
|
||||
|
||||
// writeState records everything needed to recognise and describe the process later.
|
||||
func (s *Supervisor) writeState(sh *Shard) error {
|
||||
// An adopted engine was started by an earlier agent: its command line
|
||||
// is the one its state file kept.
|
||||
cmdline := sh.cmdline
|
||||
if sh.cmd != nil {
|
||||
cmdline = sh.cmd.Args
|
||||
}
|
||||
b, err := json.Marshal(shardState{
|
||||
ShardID: sh.ID,
|
||||
ReplicaID: sh.Spec.ReplicaID,
|
||||
PID: sh.pid,
|
||||
Cmdline: sh.cmd.Args,
|
||||
Cmdline: cmdline,
|
||||
StartedAt: sh.started,
|
||||
Spec: sh.Spec,
|
||||
})
|
||||
@@ -804,6 +839,7 @@ func (s *Supervisor) Adopt(o Orphan) (*Shard, error) {
|
||||
started: o.StartedAt,
|
||||
done: make(chan struct{}),
|
||||
adopted: true,
|
||||
cmdline: o.Cmdline,
|
||||
}
|
||||
s.shards[o.ShardID] = sh
|
||||
s.mu.Unlock()
|
||||
|
||||
@@ -848,3 +848,60 @@ func eventually(t *testing.T, limit time.Duration, msg string, cond func() bool)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatAShardComputesOnSurvivesTheAgent(t *testing.T) {
|
||||
s, _ := newSupervisor(t)
|
||||
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20100}
|
||||
sh, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Stop(context.Background(), "a") })
|
||||
_ = sh
|
||||
s.SetCompute("a", engines.Compute{Backend: "CUDA", Offloaded: 49, Total: 49})
|
||||
|
||||
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
|
||||
&scriptedHealth{}, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
orphans, err := s2.FindOrphans()
|
||||
if err != nil || len(orphans) != 1 {
|
||||
t.Fatalf("%v %d", err, len(orphans))
|
||||
}
|
||||
if c := orphans[0].Spec.Compute; c == nil || c.Backend != "CUDA" || c.Offloaded != 49 {
|
||||
t.Errorf("not kept: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdoptedShardsComputeIsKeptWithoutCrashing(t *testing.T) {
|
||||
s, _ := newSupervisor(t)
|
||||
spec := engines.ShardSpec{ReplicaID: "rep-1", ShardID: "a", HTTPPort: 20100}
|
||||
if _, err := s.Start(t.Context(), "a", spec, shellCommand("sleep 30")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An agent restart: the engine is adopted by a new supervisor, which did
|
||||
// not start it and has no command of its own for it.
|
||||
s2, err := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
|
||||
&scriptedHealth{}, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
orphans, _ := s2.FindOrphans()
|
||||
if len(orphans) != 1 {
|
||||
t.Fatalf("%d orphans", len(orphans))
|
||||
}
|
||||
if _, err := s2.Adopt(orphans[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s2.Stop(context.Background(), "a") })
|
||||
s2.SetCompute("a", engines.Compute{Backend: "CPU"})
|
||||
|
||||
// And a third agent still finds it, with its command line and compute.
|
||||
s3, _ := New(Config{WorkDir: s.cfg.WorkDir, StopTimeout: 2 * time.Second},
|
||||
&scriptedHealth{}, slog.New(slog.DiscardHandler))
|
||||
again, _ := s3.FindOrphans()
|
||||
if len(again) != 1 || len(again[0].Cmdline) == 0 || again[0].Spec.Compute == nil {
|
||||
t.Fatalf("lost: %+v", again)
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
)
|
||||
|
||||
type alwaysReady struct{}
|
||||
@@ -40,7 +41,7 @@ func TestAnAdoptedEngineThatDiesIsReported(t *testing.T) {
|
||||
out := make(chan *llmgwv1.AgentMessage, 4)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
go a.watchExits(ctx, out)
|
||||
go a.watchExits(ctx, out, nil)
|
||||
time.Sleep(1500 * time.Millisecond) // seen ready
|
||||
_ = cmd.Process.Kill()
|
||||
|
||||
@@ -54,3 +55,50 @@ func TestAnAdoptedEngineThatDiesIsReported(t *testing.T) {
|
||||
t.Fatal("the adopted engine died and nothing was reported")
|
||||
}
|
||||
}
|
||||
|
||||
// Registered while its health check still ran: the ready that follows is
|
||||
// reported, or the control plane keeps the replica starting while it serves.
|
||||
func TestAShardRegisteredStartingIsReportedReady(t *testing.T) {
|
||||
sup, err := supervisor.New(supervisor.Config{WorkDir: t.TempDir(), RunAs: "-"}, alwaysReady{}, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := exec.Command("sleep", "60")
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }()
|
||||
sh, err := sup.Adopt(supervisor.Orphan{ShardID: "m-r0", ReplicaID: "m", PID: cmd.Process.Pid,
|
||||
Spec: engines.ShardSpec{ReplicaID: "m", ShardID: "m-r0", HTTPPort: 1}, Alive: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
a := &Agent{sup: sup, log: slog.New(slog.DiscardHandler)}
|
||||
pending := unready(a.runningShards())
|
||||
if !pending["m-r0"] {
|
||||
t.Fatalf("registration reported it ready already: %v", pending)
|
||||
}
|
||||
out := make(chan *llmgwv1.AgentMessage, 4)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
go a.watchExits(ctx, out, pending)
|
||||
if err := sup.WaitReady(t.Context(), sh, "http://127.0.0.1:1", "m", 5*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
select {
|
||||
case m := <-out:
|
||||
s := m.GetShard()
|
||||
if s.GetShardId() != "m-r0" || s.GetState() != agentsvc.StateToWire(supervisor.Ready) {
|
||||
t.Errorf("reported %+v", s)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("it became ready and nothing was reported")
|
||||
}
|
||||
select {
|
||||
case m := <-out:
|
||||
t.Errorf("reported twice: %+v", m.GetShard())
|
||||
case <-time.After(2500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -110,11 +111,16 @@ func (w *statusWriter) Flush() {
|
||||
}
|
||||
}
|
||||
|
||||
// notChanges are POSTs that change nothing and carry text that must not be
|
||||
// kept: asking a judge about a sentence.
|
||||
var notChanges = []string{"/api/v1/guard/judges/ask"}
|
||||
|
||||
// API records every change to the admin API: a request that is not a GET or
|
||||
// HEAD, under /api/. actor says who made it.
|
||||
func API(sink Sink, actor func(*http.Request) string, log *slog.Logger, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodGet || r.Method == http.MethodHead || !strings.HasPrefix(r.URL.Path, "/api/") {
|
||||
if r.Method == http.MethodGet || r.Method == http.MethodHead || !strings.HasPrefix(r.URL.Path, "/api/") ||
|
||||
slices.Contains(notChanges, r.URL.Path) {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -72,3 +72,13 @@ func TestABodyLargerThanWhatIsRecordedReachesTheHandlerWhole(t *testing.T) {
|
||||
t.Errorf("the handler was given %d of %d bytes", got, len(big))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAskingAJudgeKeepsNoText(t *testing.T) {
|
||||
sink := &memSink{}
|
||||
h := API(sink, func(*http.Request) string { return "ilker" }, nil, http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
|
||||
h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", "/api/v1/guard/judges/ask",
|
||||
strings.NewReader(`{"text":"something a user wrote"}`)))
|
||||
if len(sink.got) != 0 {
|
||||
t.Errorf("a judge's question was recorded: %+v", sink.got)
|
||||
}
|
||||
}
|
||||
@@ -237,6 +237,8 @@ func (w *settingsWatch) check(ctx context.Context) {
|
||||
// again: what it reads at every use, or not at all.
|
||||
var liveSettings = []string{
|
||||
"guard",
|
||||
// Stopping and starting the service (llmgw stop, llmgw start).
|
||||
"serving",
|
||||
// Sent to the nodes and this machine's firewall unit as they change.
|
||||
"firewall", "firewall_networks",
|
||||
// Read by server init, node add and backups, not by the server.
|
||||
|
||||
+56
-7
@@ -116,6 +116,13 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any)
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
// An older control plane answers a path it does not know with its
|
||||
// dashboard: a page, which read as JSON said only "invalid character '<'".
|
||||
if strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
|
||||
return fmt.Errorf("%s %s answered with a page, not the API: the control plane answering "+
|
||||
"(the leader, through a standby) does not know it — is it on an older release? "+
|
||||
"llmgw ctl status says which it runs", method, path)
|
||||
}
|
||||
return json.NewDecoder(resp.Body).Decode(out)
|
||||
}
|
||||
|
||||
@@ -168,11 +175,12 @@ func newCtlCmd() *cobra.Command {
|
||||
newJobsCmd(client, &asJSON),
|
||||
newEngineCheckCmd(),
|
||||
newEngineBuildCmd(client),
|
||||
newToolCheckCmd(),
|
||||
newToolCheckCmd(client),
|
||||
newWeightsCmd(client, &asJSON),
|
||||
newCapacityCmd(client, &asJSON),
|
||||
newQuotaCmd(client, &asJSON),
|
||||
newGuardCmd(client, &asJSON),
|
||||
newKBCmd(client, &asJSON),
|
||||
newConfigCmd(client, &asJSON),
|
||||
newAuditCmd(client, &asJSON),
|
||||
newNotifyCmd(client, &asJSON),
|
||||
@@ -225,8 +233,8 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
// What the node can run, which is the first thing to look
|
||||
// at when a model will not place and the hardware looks
|
||||
// fine.
|
||||
join(n.Engines),
|
||||
classesColumn(n), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
|
||||
enginesColumn(n),
|
||||
classesColumn(n), fabricsColumn(n), n.Replicas, n.LastSeenAgo)
|
||||
}
|
||||
_ = w.Flush()
|
||||
|
||||
@@ -443,7 +451,7 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
}
|
||||
|
||||
w := tab()
|
||||
fmt.Fprintln(w, "REPLICA\tMODEL\tSTATE\tPOOL\tNODES\tAGE\tIDLE\tENDPOINT")
|
||||
fmt.Fprintln(w, "REPLICA\tMODEL\tSTATE\tPOOL\tON\tNODES\tAGE\tIDLE\tENDPOINT")
|
||||
for _, r := range reps {
|
||||
var nodes []string
|
||||
for _, s := range r.Shards {
|
||||
@@ -461,8 +469,14 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
if r.Restart != "" {
|
||||
state += " (restart " + r.Restart + ")"
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||
r.ID, r.Model, state, r.Pool, join(nodes),
|
||||
// What the engine computes on, by its own account: the first
|
||||
// shard's, which serves.
|
||||
on := "-"
|
||||
if len(r.Shards) > 0 {
|
||||
on = r.Shards[0].Compute.Short()
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||
r.ID, r.Model, state, r.Pool, on, join(nodes),
|
||||
(time.Duration(r.AgeSec) * time.Second).String(),
|
||||
idleColumn(r.IdleSec), r.Endpoint)
|
||||
// A line rather than a column, because most replicas are not
|
||||
@@ -472,7 +486,7 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
if s.Fetch == nil {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(w, " %s: fetching %s\t\t\t\t\t\t\t\n",
|
||||
fmt.Fprintf(w, " %s: fetching %s\t\t\t\t\t\t\t\t\n",
|
||||
s.Node, fetchColumn(s.Fetch))
|
||||
}
|
||||
}
|
||||
@@ -583,7 +597,42 @@ func emitJSON(v any) error {
|
||||
// more than one or MIG instances among them: a card cut into two 3g.40gb
|
||||
// instances beside a whole card is "a100-80g-mig-3g.40gb×2,a100-80g×1", where
|
||||
// the GPUS column's 3 alone would read as three of something.
|
||||
// fabricsColumn is the node's fabrics, and when they are not all up, which
|
||||
// adapters are down: what keeps it out of replicas spanning nodes.
|
||||
func fabricsColumn(n adminapi.NodeView) string {
|
||||
s := join(n.Fabrics)
|
||||
if n.MultiNode != "" && n.MultiNode != "ready" {
|
||||
s += " (" + n.MultiNode + ")"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// enginesColumn is each engine with its version: llamacpp b10991.
|
||||
func enginesColumn(n adminapi.NodeView) string {
|
||||
if len(n.EngineVersions) == 0 {
|
||||
return join(n.Engines)
|
||||
}
|
||||
parts := make([]string, 0, len(n.Engines))
|
||||
for _, e := range n.Engines {
|
||||
if v := n.EngineVersions[e]; v != "" && v != "unknown" {
|
||||
e += " " + v
|
||||
}
|
||||
parts = append(parts, e)
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
// classesColumn is the node's GPU classes, with their compute capability
|
||||
// when they report one: 6000-24g (7.5).
|
||||
func classesColumn(n adminapi.NodeView) string {
|
||||
s := classesOnly(n)
|
||||
if len(n.ComputeCapabilities) > 0 && s != "-" {
|
||||
s += " (" + strings.Join(n.ComputeCapabilities, ",") + ")"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func classesOnly(n adminapi.NodeView) string {
|
||||
counted := len(n.ClassGPUs) > 1
|
||||
for _, c := range n.ClassGPUs {
|
||||
counted = counted || c.MIG
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAPageInsteadOfTheAPIIsSaidPlainly(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = w.Write([]byte("<!doctype html><title>llmgw</title>"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := newCtlClient(strings.TrimPrefix(srv.URL, "http://"), "")
|
||||
var out map[string]any
|
||||
err := c.get(t.Context(), "/api/v1/settings", &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "older release") || strings.Contains(err.Error(), "invalid character") {
|
||||
t.Errorf("%v", err)
|
||||
}
|
||||
}
|
||||
@@ -76,8 +76,11 @@ func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
fmt.Fprintf(out, "tags %s\n", strings.Join(kv, ", "))
|
||||
}
|
||||
if v.GuardAction != "" {
|
||||
fmt.Fprintf(out, "guard %s by policy %s (%s), %d ms\n", v.GuardAction, v.GuardPolicy,
|
||||
or(v.GuardCategory, "-"), v.GuardMs)
|
||||
what := or(v.GuardCategory, "-")
|
||||
if v.GuardRule != "" {
|
||||
what += ": " + v.GuardRule
|
||||
}
|
||||
fmt.Fprintf(out, "guard %s by policy %s (%s), %d ms\n", v.GuardAction, v.GuardPolicy, what, v.GuardMs)
|
||||
} else if v.GuardMs > 0 {
|
||||
fmt.Fprintf(out, "guard allowed, %d ms\n", v.GuardMs)
|
||||
}
|
||||
|
||||
@@ -12,9 +12,10 @@ import (
|
||||
|
||||
func newRestartCmd(client func() *ctlClient) *cobra.Command {
|
||||
var (
|
||||
models []string
|
||||
all bool
|
||||
inPlace bool
|
||||
models []string
|
||||
all bool
|
||||
inPlace bool
|
||||
surge, unavailable int
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "restart [REPLICA...]",
|
||||
@@ -31,6 +32,10 @@ func newRestartCmd(client func() *ctlClient) *cobra.Command {
|
||||
"profile edited to need more than any node has — the restart is given up\n" +
|
||||
"and the old replica keeps serving: llmgw ctl replicas says so, and why.\n\n" +
|
||||
"--in-place stops each replica first, whatever room there is.\n\n" +
|
||||
"A model's replicas are replaced one at a time. --max-surge N replaces N at\n" +
|
||||
"once, each beside itself, needing room for N more; --max-unavailable N\n" +
|
||||
"stops N at once first, needing no room and leaving N fewer serving while\n" +
|
||||
"they load. Together, the first N are stopped first and the rest surged.\n\n" +
|
||||
" llmgw ctl restart qwen2.5-7b-3f2a91c0\n" +
|
||||
" llmgw ctl restart --model qwen2.5-7b\n" +
|
||||
" llmgw ctl restart --all",
|
||||
@@ -38,7 +43,8 @@ func newRestartCmd(client func() *ctlClient) *cobra.Command {
|
||||
if len(args) == 0 && len(models) == 0 && !all {
|
||||
return errors.New("name replicas, --model, or --all")
|
||||
}
|
||||
body, _ := json.Marshal(adminapi.RestartRequest{Replicas: args, Models: models, All: all, InPlace: inPlace})
|
||||
body, _ := json.Marshal(adminapi.RestartRequest{Replicas: args, Models: models, All: all, InPlace: inPlace,
|
||||
MaxSurge: surge, MaxUnavailable: unavailable})
|
||||
var res adminapi.RestartResult
|
||||
if err := client().post(cmd.Context(), "/api/v1/restart", body, &res); err != nil {
|
||||
return err
|
||||
@@ -50,6 +56,9 @@ func newRestartCmd(client func() *ctlClient) *cobra.Command {
|
||||
c.Flags().StringArrayVar(&models, "model", nil, "every replica of this model; repeatable")
|
||||
c.Flags().BoolVar(&all, "all", false, "every ready replica in the cluster")
|
||||
c.Flags().BoolVar(&inPlace, "in-place", false, "stop each replica before starting its replacement")
|
||||
c.Flags().IntVar(&surge, "max-surge", 0, "replicas of a model replaced beside themselves at once (default 1)")
|
||||
c.Flags().IntVar(&unavailable, "max-unavailable", 0,
|
||||
"replicas of a model stopped first at once, each out while its replacement loads")
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
+22
-7
@@ -63,9 +63,10 @@ that file when the report is not enough to go on.`,
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(struct {
|
||||
Findings []doctor.Finding `json:"findings"`
|
||||
Facts doctor.Facts `json:"facts"`
|
||||
}{findings, facts})
|
||||
Findings []doctor.Finding `json:"findings"`
|
||||
Readiness doctor.Readiness `json:"readiness"`
|
||||
Facts doctor.Facts `json:"facts"`
|
||||
}{findings, doctor.ReadinessOf(findings), facts})
|
||||
}
|
||||
printDoctor(os.Stdout, facts, findings, verbose)
|
||||
for _, f := range findings {
|
||||
@@ -101,12 +102,26 @@ func printDoctor(out io.Writer, f doctor.Facts, findings []doctor.Finding, verbo
|
||||
fmt.Fprintf(out, "%s · %s %s %s · driver %s\n\n", f.Hostname, f.Host.OSID, f.Host.OSVersion,
|
||||
f.Host.Arch, orDash(f.Host.NVIDIADriver))
|
||||
|
||||
for _, x := range findings {
|
||||
fmt.Fprintf(out, "%-4s %-6s %s\n", x.Level, x.Area, x.Message)
|
||||
if x.Fix != "" && (x.Level == doctor.Fail || x.Level == doctor.Warn) {
|
||||
fmt.Fprintf(out, " → %s\n", x.Fix)
|
||||
// The node first; then what replicas spanning nodes would need, which
|
||||
// keeps a node out only when a fabric was declared for it.
|
||||
multi := false
|
||||
for _, pass := range []bool{false, true} {
|
||||
for _, x := range findings {
|
||||
if x.MultiNode() != pass {
|
||||
continue
|
||||
}
|
||||
if pass && !multi {
|
||||
fmt.Fprintln(out, "\nMulti-node (fabrics, RDMA) — for replicas spanning nodes only:")
|
||||
multi = true
|
||||
}
|
||||
fmt.Fprintf(out, "%-4s %-6s %s\n", x.Level, x.Area, x.Message)
|
||||
if x.Fix != "" && (x.Level == doctor.Fail || x.Level == doctor.Warn) {
|
||||
fmt.Fprintf(out, " → %s\n", x.Fix)
|
||||
}
|
||||
}
|
||||
}
|
||||
r := doctor.ReadinessOf(findings)
|
||||
fmt.Fprintf(out, "\nThis node: %s\nMulti-node: %s\n", r.Node, r.MultiNode)
|
||||
|
||||
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
|
||||
if len(f.GPUs) > 0 {
|
||||
|
||||
+166
-1
@@ -4,10 +4,15 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
@@ -22,6 +27,7 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
nodes int
|
||||
tp int
|
||||
perNode int
|
||||
model string
|
||||
)
|
||||
|
||||
cmd := &cobra.Command{
|
||||
@@ -31,6 +37,10 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
"engines rename and remove flags between versions. When that happens the\n" +
|
||||
"failure arrives minutes into a model load, in an engine's own error\n" +
|
||||
"message, on a node nobody was watching.\n\n" +
|
||||
"For llama.cpp on a machine with NVIDIA GPUs it then loads a small GGUF\n" +
|
||||
"(--model, or the smallest in the weights cache) with every layer on the GPU\n" +
|
||||
"and asks for one token: a build whose CUDA backend does not load computes on\n" +
|
||||
"the CPU, and that is where it is caught.\n\n" +
|
||||
"This asks the engine itself. It runs the engine's --help and checks every\n" +
|
||||
"flag llmgw would pass against it, which needs no GPU — a CLI prints its\n" +
|
||||
"help on any machine. Run it after upgrading an engine.\n\n" +
|
||||
@@ -49,6 +59,11 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
// rarely on PATH: a check that cannot find the engine reports
|
||||
// nothing rather than reporting a problem.
|
||||
cfg := engines.Config{}
|
||||
// The build the agent installed, when none is named: the newest
|
||||
// under /opt/llmgw/engines, which is where every node keeps it.
|
||||
if bin == "" && name == "llamacpp" {
|
||||
bin = installedLlama()
|
||||
}
|
||||
if bin != "" {
|
||||
switch name {
|
||||
case "llamacpp":
|
||||
@@ -135,6 +150,11 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
var deviceErr error
|
||||
if fb, ok := eng.(engines.CPUFallback); ok && help == "" {
|
||||
deviceErr = checkDevices(cmd.Context(), name, fb, out)
|
||||
// Seeing a GPU is not computing on it: a small model loaded
|
||||
// on it and asked for a token is.
|
||||
if b, ok := eng.(interface{ Binary() string }); ok && deviceErr == nil && model != "none" {
|
||||
deviceErr = computeOnGPU(cmd.Context(), b.Binary(), model, out)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintf(out, "%s: %d flags checked", name, checked)
|
||||
@@ -160,7 +180,9 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
|
||||
f := cmd.Flags()
|
||||
f.StringVar(&help, "help-output", "", "a saved --help to check against, instead of running the engine")
|
||||
f.StringVar(&bin, "bin", "", "where the engine lives, when it is not on PATH")
|
||||
f.StringVar(&bin, "bin", "", "where the engine lives (default: the build the agent installed, else PATH)")
|
||||
f.StringVar(&model, "model", "", "a GGUF file to load on the GPU and ask for a token (default: the smallest "+
|
||||
"in this node's weights cache; none skips it)")
|
||||
f.IntVar(&nodes, "nodes", 2, "shape the command for a replica of this many nodes")
|
||||
f.IntVar(&tp, "tp", 8, "tensor parallel degree")
|
||||
f.IntVar(&perNode, "gpus-per-node", 4, "GPUs each node contributes")
|
||||
@@ -286,3 +308,146 @@ func checkDevices(ctx context.Context, name string, fb engines.CPUFallback, out
|
||||
fmt.Fprintf(out, "computes on: %s\n", strings.Join(devs, "; "))
|
||||
return nil
|
||||
}
|
||||
|
||||
// llamaRoot is where the agent installs llama.cpp builds.
|
||||
var llamaRoot = "/opt/llmgw/engines/llamacpp"
|
||||
|
||||
// installedLlama is the llama-server the agent installed here: under
|
||||
// llamaRoot/<version-variant>/llama-<build>/, or one level up for older
|
||||
// layouts; on a machine with NVIDIA GPUs a CUDA build first, then the newest.
|
||||
func installedLlama() string {
|
||||
var found []string
|
||||
for _, pat := range []string{"*/llama-server", "*/*/llama-server"} {
|
||||
m, _ := filepath.Glob(filepath.Join(llamaRoot, pat))
|
||||
found = append(found, m...)
|
||||
}
|
||||
if len(found) == 0 {
|
||||
return ""
|
||||
}
|
||||
sort.Strings(found)
|
||||
if _, err := exec.LookPath("nvidia-smi"); err == nil {
|
||||
for i := len(found) - 1; i >= 0; i-- {
|
||||
if strings.Contains(found[i], "cuda") {
|
||||
return found[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
return found[len(found)-1]
|
||||
}
|
||||
|
||||
// weightDirs are where a node keeps models it has fetched.
|
||||
var weightDirs = []string{"/var/lib/llmgw-weights", "/var/lib/llmgw-models", "/srv/models"}
|
||||
|
||||
// smallestGGUF is the smallest GGUF file under dirs, for a check that only
|
||||
// needs something to load.
|
||||
func smallestGGUF(dirs []string) string {
|
||||
best, size := "", int64(0)
|
||||
for _, d := range dirs {
|
||||
_ = filepath.WalkDir(d, func(p string, e os.DirEntry, err error) error {
|
||||
if err != nil || e.IsDir() || !strings.HasSuffix(strings.ToLower(p), ".gguf") ||
|
||||
strings.Contains(strings.ToLower(filepath.Base(p)), "mmproj") {
|
||||
return nil
|
||||
}
|
||||
if info, err := e.Info(); err == nil && (best == "" || info.Size() < size) {
|
||||
best, size = p, info.Size()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// computeOnGPU loads a model with every layer on the GPU, asks for one
|
||||
// token, and reads from llama.cpp's own output where the layers went.
|
||||
func computeOnGPU(ctx context.Context, bin, model string, out io.Writer) error {
|
||||
if _, err := exec.LookPath("nvidia-smi"); err != nil {
|
||||
return nil
|
||||
}
|
||||
if model == "" {
|
||||
if model = smallestGGUF(weightDirs); model == "" {
|
||||
fmt.Fprintln(out, "no GGUF in this node's weights cache to compute with; --model PATH runs one")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
port := ln.Addr().(*net.TCPAddr).Port
|
||||
_ = ln.Close()
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
|
||||
defer cancel()
|
||||
var logs syncBuffer
|
||||
c := exec.CommandContext(ctx, bin, "-m", model, "--host", "127.0.0.1", "--port", strconv.Itoa(port), //nolint:gosec // the engine's own binary
|
||||
"--n-gpu-layers", "999", "--ctx-size", "512", "--parallel", "1",
|
||||
// Where the layers went is said only at this verbosity.
|
||||
"-lv", "4")
|
||||
c.Stdout, c.Stderr = &logs, &logs
|
||||
if err := c.Start(); err != nil {
|
||||
return fmt.Errorf("starting %s: %w", bin, err)
|
||||
}
|
||||
defer func() { _ = c.Process.Kill(); _ = c.Wait() }()
|
||||
base := "http://127.0.0.1:" + strconv.Itoa(port)
|
||||
start := time.Now()
|
||||
for {
|
||||
resp, err := http.Get(base + "/health") //nolint:gosec,noctx // loopback, bounded by the loop
|
||||
if err == nil {
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return fmt.Errorf("%s did not come up with %s: %s", bin, filepath.Base(model), tailLines(logs.String(), 5))
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodPost, base+"/completion",
|
||||
strings.NewReader(`{"prompt":"Hello","n_predict":1}`))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("asking it for a token: %w", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
took := time.Since(start)
|
||||
comp, said := engines.ReadLlamaCompute(strings.Split(logs.String(), "\n"))
|
||||
if !said {
|
||||
fmt.Fprintf(out, "computed a token with %s, but llama.cpp did not say where\n", filepath.Base(model))
|
||||
return nil
|
||||
}
|
||||
if comp.OnTheCPU() {
|
||||
return fmt.Errorf("%s computed on the CPU though every layer was asked for on the GPU (%d of %d offloaded): "+
|
||||
"its CUDA backend did not load; ldd on its libggml-cuda.so names the missing library",
|
||||
filepath.Base(model), comp.Offloaded, comp.Total)
|
||||
}
|
||||
fmt.Fprintf(out, "computed a token on %s: %s, %d of %d layers offloaded, in %s\n", comp.Backend,
|
||||
strings.Join(comp.Devices, "; "), comp.Offloaded, comp.Total, took.Round(100*time.Millisecond))
|
||||
return nil
|
||||
}
|
||||
|
||||
func tailLines(s string, n int) string {
|
||||
l := strings.Split(strings.TrimSpace(s), "\n")
|
||||
if len(l) > n {
|
||||
l = l[len(l)-n:]
|
||||
}
|
||||
return strings.Join(l, " | ")
|
||||
}
|
||||
|
||||
// syncBuffer is a buffer two writers share.
|
||||
type syncBuffer struct {
|
||||
mu sync.Mutex
|
||||
b strings.Builder
|
||||
}
|
||||
|
||||
func (s *syncBuffer) Write(p []byte) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.b.Write(p)
|
||||
}
|
||||
|
||||
func (s *syncBuffer) String() string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.b.String()
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestMain lets the test binary stand in for llama-server: run with
|
||||
// LLMGW_FAKE_LLAMA set, it prints what llama.cpp prints and answers.
|
||||
func TestMain(m *testing.M) {
|
||||
if mode := os.Getenv("LLMGW_FAKE_LLAMA"); mode != "" {
|
||||
fakeLlama(mode)
|
||||
return
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func fakeLlama(mode string) {
|
||||
port := ""
|
||||
for i, a := range os.Args {
|
||||
if a == "--port" && i+1 < len(os.Args) {
|
||||
port = os.Args[i+1]
|
||||
}
|
||||
}
|
||||
if mode == "gpu" {
|
||||
fmt.Println("llama_model_load_from_file_impl: using device CUDA0 (Quadro RTX 6000) - 23800 MiB free")
|
||||
fmt.Println("load_tensors: offloaded 25/25 layers to GPU")
|
||||
} else {
|
||||
fmt.Println("warning: no usable GPU found, --gpu-layers option will be ignored")
|
||||
fmt.Println("load_tensors: offloaded 0/25 layers to GPU")
|
||||
}
|
||||
http.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })
|
||||
http.HandleFunc("/completion", func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{"content":"!"}`)) })
|
||||
_ = http.ListenAndServe("127.0.0.1:"+port, nil) //nolint:gosec // a test's stand-in
|
||||
}
|
||||
|
||||
func TestEngineCheckComputesATokenOnTheGPU(t *testing.T) {
|
||||
// nvidia-smi present, so the check runs.
|
||||
dir := t.TempDir()
|
||||
smi := filepath.Join(dir, "nvidia-smi")
|
||||
if err := os.WriteFile(smi, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
model := filepath.Join(dir, "tiny.gguf")
|
||||
_ = os.WriteFile(model, []byte("GGUF"), 0o644)
|
||||
self, _ := os.Executable()
|
||||
|
||||
t.Setenv("LLMGW_FAKE_LLAMA", "gpu")
|
||||
var out bytes.Buffer
|
||||
if err := computeOnGPU(t.Context(), self, model, &out); err != nil || !strings.Contains(out.String(), "on CUDA") ||
|
||||
!strings.Contains(out.String(), "25 of 25") {
|
||||
t.Fatalf("on the GPU: %v %s", err, out.String())
|
||||
}
|
||||
|
||||
t.Setenv("LLMGW_FAKE_LLAMA", "cpu")
|
||||
out.Reset()
|
||||
if err := computeOnGPU(t.Context(), self, model, &out); err == nil || !strings.Contains(err.Error(), "computed on the CPU") {
|
||||
t.Fatalf("a build that falls back to the CPU: %v %s", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineCheckFindsTheBuildTheAgentInstalled(t *testing.T) {
|
||||
old := llamaRoot
|
||||
llamaRoot = t.TempDir()
|
||||
t.Cleanup(func() { llamaRoot = old })
|
||||
if got := installedLlama(); got != "" {
|
||||
t.Fatalf("nothing installed, found %q", got)
|
||||
}
|
||||
for _, p := range []string{"b10991-cpu/llama-b10991/llama-server", "b10991-cuda-13.3/llama-b10991/llama-server"} {
|
||||
full := filepath.Join(llamaRoot, p)
|
||||
_ = os.MkdirAll(filepath.Dir(full), 0o755)
|
||||
_ = os.WriteFile(full, []byte("x"), 0o755)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
_ = os.WriteFile(filepath.Join(dir, "nvidia-smi"), []byte("#!/bin/sh\n"), 0o755)
|
||||
t.Setenv("PATH", dir)
|
||||
if got := installedLlama(); !strings.Contains(got, "cuda-13.3/llama-b10991/llama-server") {
|
||||
t.Errorf("with GPUs, not the CUDA build: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -3,11 +3,13 @@ package cli
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
)
|
||||
|
||||
func newGuardCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
@@ -24,6 +26,35 @@ func newGuardCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
"Targets are written org:Acme, team:Acme/ml, project:Acme/ml/training (or\n" +
|
||||
"with IDs), key:KEY-ID or user:USER-ID. An attachment applies at once.",
|
||||
}
|
||||
c.AddCommand(newGuardJudgeCmd(client, asJSON))
|
||||
for _, sw := range []struct {
|
||||
use, short string
|
||||
on bool
|
||||
}{
|
||||
{"on", "Turn the guardrails on: every policy checks again", true},
|
||||
{"off", "Turn the guardrails off: no request is checked, whatever the policies say", false},
|
||||
} {
|
||||
var note string
|
||||
cmd := &cobra.Command{
|
||||
Use: sw.use,
|
||||
Short: sw.short,
|
||||
Long: "guard.enabled in the cluster's settings: in force on every control plane\n" +
|
||||
"within seconds, without a restart. The policies and where they are attached\n" +
|
||||
"stay as they are, and come back as they were when turned on again. While\n" +
|
||||
"off, llmgw ctl status says so.",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if note == "" {
|
||||
note = "guardrails " + sw.use
|
||||
}
|
||||
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
|
||||
return doc.SetYAML([]string{"guard", "enabled"}, strconv.FormatBool(sw.on))
|
||||
})
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version")
|
||||
c.AddCommand(cmd)
|
||||
}
|
||||
c.AddCommand(&cobra.Command{
|
||||
Use: "list",
|
||||
Short: "The policies, and where each is attached",
|
||||
@@ -37,6 +68,9 @@ func newGuardCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
if !v.Enabled {
|
||||
fmt.Fprintln(out, "The guardrails are OFF: no request is checked (llmgw ctl guard on).")
|
||||
}
|
||||
fmt.Fprintf(out, "Policies: %s\n", strings.Join(v.Policies, ", "))
|
||||
def := strings.Join(v.Default, ", ")
|
||||
if def == "" {
|
||||
|
||||
@@ -0,0 +1,385 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
)
|
||||
|
||||
func newGuardJudgeCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
Use: "judge",
|
||||
Short: "The judge models the policies ask: whether each can answer, and serving one",
|
||||
Long: "A policy with a judge has a model read what patterns cannot. The judge must\n" +
|
||||
"be in the catalog under the name the policy gives, as a judge (role:\n" +
|
||||
"guard), with a replica ready; otherwise what the policy judges is refused\n" +
|
||||
"(fail: closed, guard_unavailable) or served unjudged (fail: open).\n\n" +
|
||||
"A policy may name several, asked in turn (judge.models): a judge on a GPU\n" +
|
||||
"first and one on CPUs after it is the GPU's speed while it is there, and\n" +
|
||||
"the CPUs' when it is not.\n\n" +
|
||||
"Judges llmgw knows where to fetch, which a policy may name and the leader\n" +
|
||||
"then fetches and serves by itself: " + knownJudgeNames() + ".",
|
||||
}
|
||||
c.AddCommand(newGuardJudgeStatusCmd(client, asJSON), newGuardJudgeEnableCmd(client),
|
||||
newGuardJudgeUseCmd(client), newGuardJudgeAskCmd(client, asJSON))
|
||||
return c
|
||||
}
|
||||
|
||||
func newGuardJudgeStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Each judge the policies ask, and what is wrong with it",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
var js []adminapi.JudgeView
|
||||
if err := client().get(cmd.Context(), "/api/v1/guard/judges", &js); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return emitJSON(js)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
if len(js) == 0 {
|
||||
fmt.Fprintln(out, "no policy asks a judge")
|
||||
return nil
|
||||
}
|
||||
w := tabTo(out)
|
||||
fmt.Fprintln(w, "JUDGE\tPOLICIES\tIN CATALOG\tAS JUDGE\tREADY\tSTATE")
|
||||
for _, j := range js {
|
||||
state := "answering"
|
||||
if j.Problem != "" {
|
||||
state = j.Problem
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d/%d\t%s\n", j.Model, strings.Join(j.Policies, ","),
|
||||
yesNo(j.InCatalog), yesNo(j.Judge), j.Ready, j.Wanted, state)
|
||||
}
|
||||
for _, j := range js {
|
||||
if len(j.Then) > 0 {
|
||||
fmt.Fprintf(w, "\n%s gives no verdict: %s is asked instead", j.Model, strings.Join(j.Then, ", else "))
|
||||
}
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
strays, _ := strayJudges(cmd, client, js)
|
||||
sayStrays(out, strays)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// strayJudges are models whose weights are a judge's, under a name no policy
|
||||
// asks: a judge the policies moved on from, or one served from the mirror
|
||||
// page as a model clients use, as before that page knew better.
|
||||
func strayJudges(cmd *cobra.Command, client func() *ctlClient, js []adminapi.JudgeView) ([]adminapi.ModelView, error) {
|
||||
var models []adminapi.ModelView
|
||||
if err := client().get(cmd.Context(), "/api/v1/models", &models); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
asked := map[string]bool{}
|
||||
for _, j := range js {
|
||||
asked[j.Model] = true
|
||||
}
|
||||
var out []adminapi.ModelView
|
||||
for _, m := range models {
|
||||
if _, judge := guard.FormatOfWeights(m.Source); !asked[m.Name] && judge {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sayStrays says what each stray judge is and how to be rid of it.
|
||||
func sayStrays(out io.Writer, strays []adminapi.ModelView) {
|
||||
for _, m := range strays {
|
||||
if m.Role == catalog.RoleGuard {
|
||||
fmt.Fprintf(out, "\n%s is a judge no policy asks, still running: llmgw ctl model remove %s\n", m.Name, m.Name)
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(out, "\n%s looks like a judge's weights but is served as a model clients use, and no "+
|
||||
"policy asks it: llmgw ctl model remove %s\n", m.Name, m.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func newGuardJudgeEnableCmd(client func() *ctlClient) *cobra.Command {
|
||||
var (
|
||||
model, repo, file, format, gpuClass string
|
||||
replicas, cores int
|
||||
gpu, replace, detach bool
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "enable",
|
||||
Short: "Serve a judge: Qwen3Guard-Gen 0.6B on CPUs, or with --gpu the 4B on a GPU",
|
||||
Long: "Fetches a judge's weights into the mirror (once; what is there already, its\n" +
|
||||
"checksums matching, is not fetched again), adds it to the catalog as a judge\n" +
|
||||
"(role: guard) and waits for a replica to answer. Which judge: --model, or\n" +
|
||||
"with --gpu " + defaultGPUJudge + ", or else the first the policies ask that is\n" +
|
||||
"missing, or else " + defaultJudge + ".\n\n" +
|
||||
"A judge on CPUs takes no card. One on a GPU takes one card, as a standard\n" +
|
||||
"model: from nothing more important, so it waits while the cards are busy;\n" +
|
||||
"put a judge on CPUs after it in the policies (llmgw ctl guard judge use) so\n" +
|
||||
"there is always one to ask.\n\n" +
|
||||
"Known judges: " + knownJudgeNames() + ". Another is served with --repo,\n" +
|
||||
"--file and --format. A model of that name already in the catalog as a\n" +
|
||||
"model clients use is refused; --replace removes it first. An offline\n" +
|
||||
"cluster gets its judge from its bundle with server init.",
|
||||
Example: " llmgw ctl guard judge enable\n" +
|
||||
" llmgw ctl guard judge enable --gpu\n" +
|
||||
" llmgw ctl guard judge use " + defaultGPUJudge + " " + defaultJudge,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
ctx := cmd.Context()
|
||||
out := cmd.OutOrStdout()
|
||||
var js []adminapi.JudgeView
|
||||
if err := client().get(ctx, "/api/v1/guard/judges", &js); err != nil {
|
||||
return err
|
||||
}
|
||||
name := model
|
||||
if name == "" && gpu {
|
||||
name = defaultGPUJudge
|
||||
}
|
||||
if name == "" {
|
||||
for _, j := range js {
|
||||
if !j.InCatalog || !j.Judge {
|
||||
name = j.Model
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if name == "" {
|
||||
name = defaultJudge
|
||||
}
|
||||
|
||||
k, known := knownJudgeNamed(name)
|
||||
switch {
|
||||
case repo != "":
|
||||
f := format
|
||||
if f == "" {
|
||||
f, _ = guard.FormatOfWeights(repo + "/" + file)
|
||||
}
|
||||
if f == "" || !guard.KnownFormat(f) {
|
||||
return fmt.Errorf("say how its answers are read: --format %s", strings.Join(guard.Formats, "|"))
|
||||
}
|
||||
k = knownJudge{Name: name, Repo: repo, File: file, Format: f, GPU: gpu}
|
||||
case !known:
|
||||
return fmt.Errorf("%s is not a judge llmgw knows where to fetch (%s): give --repo, --file and --format", name, knownJudgeNames())
|
||||
}
|
||||
if gpu {
|
||||
k.GPU = true
|
||||
}
|
||||
|
||||
var models []adminapi.ModelView
|
||||
if err := client().get(ctx, "/api/v1/models", &models); err != nil {
|
||||
return err
|
||||
}
|
||||
if i := slices.IndexFunc(models, func(m adminapi.ModelView) bool { return m.Name == name }); i >= 0 {
|
||||
if models[i].Role == catalog.RoleGuard {
|
||||
fmt.Fprintf(out, "%s is in the catalog as a judge already; llmgw ctl guard judge status says how it is\n", name)
|
||||
return nil
|
||||
}
|
||||
if !replace {
|
||||
return fmt.Errorf("%s is in the catalog as a model clients use, not as a judge: "+
|
||||
"--replace removes it and serves it again as a judge", name)
|
||||
}
|
||||
var res map[string]string
|
||||
if err := client().delete(ctx, "/api/v1/models/"+name, &res); err != nil {
|
||||
return fmt.Errorf("removing %s: %w", name, err)
|
||||
}
|
||||
fmt.Fprintf(out, "%s removed\n", name)
|
||||
}
|
||||
|
||||
opts := judgeServeOptions{Replicas: replicas, Cores: cores, GPUClass: gpuClass}
|
||||
if k.GPU && opts.GPUClass == "" {
|
||||
var s adminapi.Summary
|
||||
if err := client().get(ctx, "/api/v1/summary", &s); err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.GPUClass = roomiestGPUClass(s.GPUs); opts.GPUClass == "" {
|
||||
return fmt.Errorf("this cluster has no GPU for %s: serve %s on CPUs instead", name, defaultJudge)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(out, "serving %s\n", k)
|
||||
body, _ := json.Marshal(k.serveRequest(opts))
|
||||
var job modelserve.Job
|
||||
if err := client().patient(30*time.Second).post(ctx, "/api/v1/models/serve", body, &job); err != nil {
|
||||
return err
|
||||
}
|
||||
if detach {
|
||||
return detached(cmd, job)
|
||||
}
|
||||
if err := followJob(cmd, client, job, "is the judge"); err != nil {
|
||||
return err
|
||||
}
|
||||
if !slices.ContainsFunc(js, func(j adminapi.JudgeView) bool { return j.Model == name }) {
|
||||
fmt.Fprintf(out, "\nno policy asks %s yet: llmgw ctl guard judge use %s", name, name)
|
||||
if k.GPU {
|
||||
fmt.Fprintf(out, " %s", defaultJudge)
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
if strays, _ := strayJudges(cmd, client, js); len(strays) > 0 {
|
||||
sayStrays(out, slices.DeleteFunc(strays, func(m adminapi.ModelView) bool { return m.Name == name }))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
f := c.Flags()
|
||||
f.StringVar(&model, "model", "", "the judge to serve: "+knownJudgeNames()+", or a name for --repo's")
|
||||
f.BoolVar(&gpu, "gpu", false, "serve it on a GPU ("+defaultGPUJudge+" when no --model)")
|
||||
f.StringVar(&gpuClass, "gpu-class", "", "the GPUs it runs on (default: the class with the most free cards)")
|
||||
f.StringVar(&repo, "repo", "", "the registry repository of a judge llmgw does not know")
|
||||
f.StringVar(&file, "file", "", "the GGUF file in --repo")
|
||||
f.StringVar(&format, "format", "", "how --repo's judge answers: "+strings.Join(guard.Formats, ", "))
|
||||
f.IntVar(&replicas, "replicas", 0, "replicas (default: 2 on CPUs, as every judged request waits on one; 1 on a GPU)")
|
||||
f.IntVar(&cores, "cores", 4, "CPU cores a replica on CPUs computes on")
|
||||
f.BoolVar(&replace, "replace", false, "remove a model of that name that is not a judge first")
|
||||
f.BoolVar(&detach, "detach", false, "start it and return; llmgw ctl jobs follows it")
|
||||
return c
|
||||
}
|
||||
|
||||
// roomiestGPUClass is the class with the most free cards, else the most
|
||||
// cards; empty when the cluster has none.
|
||||
func roomiestGPUClass(classes []adminapi.ClassUse) string {
|
||||
best := -1
|
||||
for i, c := range classes {
|
||||
if c.Total == 0 {
|
||||
continue
|
||||
}
|
||||
if best < 0 || c.Free() > classes[best].Free() || (c.Free() == classes[best].Free() && c.Total > classes[best].Total) {
|
||||
best = i
|
||||
}
|
||||
}
|
||||
if best < 0 {
|
||||
return ""
|
||||
}
|
||||
return classes[best].Class
|
||||
}
|
||||
|
||||
func newGuardJudgeUseCmd(client func() *ctlClient) *cobra.Command {
|
||||
var policies []string
|
||||
var note string
|
||||
c := &cobra.Command{
|
||||
Use: "use JUDGE...",
|
||||
Short: "Set the judges the policies ask, in the order they are asked",
|
||||
Long: "Sets judge.models of every policy that has a judge, or of those given with\n" +
|
||||
"--policy: the first judge that answers gives the verdict, and the next is\n" +
|
||||
"asked when one has no replica serving, fails, or does not answer within\n" +
|
||||
"the policy's latency_budget. A judge on a GPU first and one on CPUs after\n" +
|
||||
"it is the GPU's speed while it is there, and the CPUs' when it is not.\n\n" +
|
||||
"A judge llmgw knows (" + knownJudgeNames() + ")\n" +
|
||||
"that is not in the catalog is fetched and served by the leader; another\n" +
|
||||
"is served first with llmgw ctl guard judge enable --repo. In force on\n" +
|
||||
"every control plane within seconds.",
|
||||
Example: " llmgw ctl guard judge use " + defaultJudge + "\n" +
|
||||
" llmgw ctl guard judge use " + defaultGPUJudge + " " + defaultJudge + "\n" +
|
||||
" llmgw ctl guard judge use " + defaultGPUJudge + " " + defaultJudge + " --policy strict",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
for i, a := range args {
|
||||
if slices.Contains(args[:i], a) {
|
||||
return fmt.Errorf("%s is named twice", a)
|
||||
}
|
||||
}
|
||||
if note == "" {
|
||||
note = "judges: " + strings.Join(args, ", else ")
|
||||
}
|
||||
list := "[" + strings.Join(args, ", ") + "]"
|
||||
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
|
||||
names := policies
|
||||
if len(names) == 0 {
|
||||
for _, p := range doc.MapKeys([]string{"guard", "policies"}) {
|
||||
if doc.Node([]string{"guard", "policies", p, "judge"}) != nil {
|
||||
names = append(names, p)
|
||||
}
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return errors.New("no policy has a judge: name one with --policy, and it is given one " +
|
||||
"that blocks what the judge finds unsafe in requests (judge.input: block)")
|
||||
}
|
||||
}
|
||||
for _, p := range names {
|
||||
base := []string{"guard", "policies", p}
|
||||
if doc.Node(base) == nil {
|
||||
return fmt.Errorf("there is no policy %s (llmgw ctl guard list)", p)
|
||||
}
|
||||
judge := append(slices.Clone(base), "judge")
|
||||
if doc.Node(judge) == nil {
|
||||
if err := doc.SetYAML(append(slices.Clone(judge), "input"), "block"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
doc.Unset(append(slices.Clone(judge), "model"))
|
||||
if err := doc.SetYAML(append(slices.Clone(judge), "models"), list); err != nil {
|
||||
return err
|
||||
}
|
||||
cats := append(slices.Clone(judge), "categories")
|
||||
var have []string
|
||||
if n := doc.Node(cats); n != nil {
|
||||
for _, c := range n.Content {
|
||||
have = append(have, c.Value)
|
||||
}
|
||||
}
|
||||
if next := categoriesFor(args, have); next != nil {
|
||||
if err := doc.SetYAML(cats, "["+strings.Join(next, ", ")+"]"); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s: categories [%s] are not what %s name; it counts [%s] instead\n",
|
||||
p, strings.Join(have, ", "), strings.Join(args, " or "), strings.Join(next, ", "))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
},
|
||||
}
|
||||
c.Flags().StringSliceVar(&policies, "policy", nil, "the policies to change (default: every one with a judge)")
|
||||
c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version")
|
||||
return c
|
||||
}
|
||||
|
||||
func newGuardJudgeAskCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
var model, answer string
|
||||
c := &cobra.Command{
|
||||
Use: "ask TEXT",
|
||||
Short: "Have a judge read a text and say what it answered",
|
||||
Long: "For finding out why a request was refused: the judge reads TEXT as a user's\n" +
|
||||
"message (with --answer, ANSWER as the reply to it), past the cache, and its\n" +
|
||||
"answer is shown as it gave it and as the gateway reads it — \"unsafe\" and\n" +
|
||||
"the hazards it named, S6 Specialized Advice and the like. Nothing is kept:\n" +
|
||||
"not in the audit, the usage or the logs.",
|
||||
Example: " llmgw ctl guard judge ask 'How do I sort a list in Python?'\n" +
|
||||
" llmgw ctl guard judge ask 'Write a poem' --answer 'Roses are red'",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
body, _ := json.Marshal(adminapi.JudgeAskRequest{Model: model, Text: args[0], Answer: answer})
|
||||
var v adminapi.JudgeAskView
|
||||
if err := client().patient(45*time.Second).post(cmd.Context(), "/api/v1/guard/judges/ask", body, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return emitJSON(v)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "%s: %s", v.Model, v.Verdict)
|
||||
if len(v.Categories) > 0 {
|
||||
fmt.Fprintf(out, " (%s)", strings.Join(v.Categories, ", "))
|
||||
}
|
||||
fmt.Fprintf(out, ", %d ms\nit answered: %q\n", v.TookMs, v.Raw)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(&model, "model", "", "the judge (default: the one the policies ask)")
|
||||
c.Flags().StringVar(&answer, "answer", "", "an assistant's reply to TEXT, judged as output is")
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
// judgeKeeper fetches and serves, on the leader, each judge llmgw knows that
|
||||
// a policy names and the catalog does not have: naming qwen3guard-4b in a
|
||||
// policy is all it takes to have one, and a judge removed by mistake comes
|
||||
// back. A judge it does not know is left to llmgw ctl guard judge enable
|
||||
// --repo, said in guard judge status.
|
||||
type judgeKeeper struct {
|
||||
library func() guard.Config
|
||||
models func() []catalog.Model
|
||||
serve interface {
|
||||
Start(modelserve.Request) (modelserve.Job, error)
|
||||
Jobs() []modelserve.Job
|
||||
}
|
||||
leader func() bool
|
||||
gpus func() []adminapi.ClassUse
|
||||
log *slog.Logger
|
||||
now func() time.Time
|
||||
|
||||
// tried is when each judge was last asked for, so one that cannot be
|
||||
// fetched is not asked for again on every look.
|
||||
tried map[string]time.Time
|
||||
}
|
||||
|
||||
// judgeRetry is how long a judge whose serving failed waits before it is
|
||||
// asked for again.
|
||||
const judgeRetry = 10 * time.Minute
|
||||
|
||||
func (k *judgeKeeper) run(ctx context.Context, every time.Duration) {
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
k.check()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (k *judgeKeeper) check() {
|
||||
if k.leader != nil && !k.leader() {
|
||||
return
|
||||
}
|
||||
lib := k.library()
|
||||
if !lib.On() {
|
||||
return
|
||||
}
|
||||
if k.tried == nil {
|
||||
k.tried = map[string]time.Time{}
|
||||
}
|
||||
now := time.Now
|
||||
if k.now != nil {
|
||||
now = k.now
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, m := range k.models() {
|
||||
have[m.Name] = true
|
||||
}
|
||||
busy := map[string]bool{}
|
||||
for _, j := range k.serve.Jobs() {
|
||||
if j.State == modelserve.Running {
|
||||
busy[j.Model] = true
|
||||
busy[j.Request.Name] = true
|
||||
}
|
||||
}
|
||||
var wanted []string
|
||||
for _, p := range lib.Policies {
|
||||
for _, j := range p.Judge.Judges() {
|
||||
if !slices.Contains(wanted, j) {
|
||||
wanted = append(wanted, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.Sort(wanted)
|
||||
for _, name := range wanted {
|
||||
kj, known := knownJudgeNamed(name)
|
||||
if have[name] || !known || busy[name] {
|
||||
continue
|
||||
}
|
||||
if at, ok := k.tried[name]; ok && now().Sub(at) < judgeRetry {
|
||||
continue
|
||||
}
|
||||
k.tried[name] = now()
|
||||
var opts judgeServeOptions
|
||||
if kj.GPU {
|
||||
if opts.GPUClass = roomiestGPUClass(k.gpus()); opts.GPUClass == "" {
|
||||
k.log.Warn("a policy names a judge that runs on a GPU, and this cluster has none",
|
||||
"judge", name, "instead", defaultJudge)
|
||||
continue
|
||||
}
|
||||
}
|
||||
req := kj.serveRequest(opts)
|
||||
req.By = "llmgw (a guard policy names it)"
|
||||
job, err := k.serve.Start(req)
|
||||
if err != nil {
|
||||
k.log.Error("could not start serving a judge a policy names", "judge", name, "err", err)
|
||||
continue
|
||||
}
|
||||
k.log.Info("serving a judge a policy names, which the catalog did not have",
|
||||
"judge", name, "repo", kj.Repo, "file", kj.File, "job", job.ID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
type fakeServe struct {
|
||||
started []modelserve.Request
|
||||
running []modelserve.Job
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeServe) Start(r modelserve.Request) (modelserve.Job, error) {
|
||||
if f.err != nil {
|
||||
return modelserve.Job{}, f.err
|
||||
}
|
||||
f.started = append(f.started, r)
|
||||
return modelserve.Job{ID: "j", Model: r.Name, Request: r, State: modelserve.Running}, nil
|
||||
}
|
||||
func (f *fakeServe) Jobs() []modelserve.Job { return f.running }
|
||||
|
||||
func TestAJudgeAPolicyNamesIsFetchedWhenTheCatalogLacksIt(t *testing.T) {
|
||||
lib := guard.Config{Policies: map[string]guard.Policy{
|
||||
"baseline": {Judge: &guard.JudgePolicy{Models: []string{"qwen3guard-4b", "qwen3guard-0.6b"}, Input: "block"}},
|
||||
"custom": {Judge: &guard.JudgePolicy{Model: "our-own-judge", Input: "block"}},
|
||||
}}
|
||||
inCatalog := []catalog.Model{{Name: "qwen3guard-0.6b"}}
|
||||
now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)
|
||||
fs := &fakeServe{}
|
||||
leader := true
|
||||
k := &judgeKeeper{
|
||||
library: func() guard.Config { return lib },
|
||||
models: func() []catalog.Model { return inCatalog },
|
||||
serve: fs,
|
||||
leader: func() bool { return leader },
|
||||
gpus: func() []adminapi.ClassUse { return []adminapi.ClassUse{{Class: "6000-24g", Total: 1, Used: 1}} },
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
now: func() time.Time { return now },
|
||||
}
|
||||
|
||||
k.check()
|
||||
// The GPU judge alone: the CPU one is there, and one llmgw does not know
|
||||
// is not guessed at.
|
||||
if len(fs.started) != 1 || fs.started[0].Name != "qwen3guard-4b" || fs.started[0].GPUClass != "6000-24g" ||
|
||||
fs.started[0].Class != catalog.ClassStandard || fs.started[0].GuardFormat != "qwen3guard" || fs.started[0].CPU {
|
||||
t.Fatalf("started %+v", fs.started)
|
||||
}
|
||||
|
||||
// Being fetched: not asked for again.
|
||||
fs.running = []modelserve.Job{{Model: "qwen3guard-4b", State: modelserve.Running}}
|
||||
k.check()
|
||||
if len(fs.started) != 1 {
|
||||
t.Errorf("asked for again while it is being fetched: %d", len(fs.started))
|
||||
}
|
||||
// Failed: asked for again only after a while.
|
||||
fs.running = nil
|
||||
now = now.Add(time.Minute)
|
||||
k.check()
|
||||
if len(fs.started) != 1 {
|
||||
t.Error("asked for again at once after it failed")
|
||||
}
|
||||
now = now.Add(judgeRetry)
|
||||
k.check()
|
||||
if len(fs.started) != 2 {
|
||||
t.Error("never asked for again")
|
||||
}
|
||||
|
||||
// Only the leader fetches, and nothing while the guardrails are off.
|
||||
leader = false
|
||||
now = now.Add(judgeRetry)
|
||||
k.check()
|
||||
leader = true
|
||||
off := false
|
||||
lib.Enabled = &off
|
||||
k.check()
|
||||
if len(fs.started) != 2 {
|
||||
t.Errorf("a standby or the guardrails off fetched: %d", len(fs.started))
|
||||
}
|
||||
|
||||
// A CPU judge on a cluster without GPUs is served on CPUs, two of it.
|
||||
lib = guard.Config{Policies: map[string]guard.Policy{"b": {Judge: &guard.JudgePolicy{Model: "qwen3guard-0.6b", Input: "block"}}}}
|
||||
inCatalog = nil
|
||||
k.gpus = func() []adminapi.ClassUse { return nil }
|
||||
k.check()
|
||||
if last := fs.started[len(fs.started)-1]; last.Name != "qwen3guard-0.6b" || !last.CPU || last.Replicas != 2 {
|
||||
t.Errorf("CPU judge: %+v", last)
|
||||
}
|
||||
fs.err = errors.New("the mirror is down")
|
||||
lib = guard.Config{Policies: map[string]guard.Policy{"b": {Judge: &guard.JudgePolicy{Model: "qwen3guard-8b", Input: "block"}}}}
|
||||
k.check() // no GPU, so not started, and no panic on the error path either
|
||||
}
|
||||
|
||||
// A policy moved from Llama Guard's codes to Qwen3Guard would count nothing:
|
||||
// its categories become the judge's own.
|
||||
func TestCategoriesFollowTheJudges(t *testing.T) {
|
||||
lg := []string{"S1", "S2", "S10"}
|
||||
if got := categoriesFor([]string{"qwen3guard-0.6b"}, lg); !slices.Equal(got, baselineCategories["qwen3guard"]) {
|
||||
t.Errorf("Llama Guard's codes for Qwen3Guard: %v", got)
|
||||
}
|
||||
if got := categoriesFor([]string{"qwen3guard-4b", "qwen3guard-0.6b"}, []string{"Violent", "Jailbreak"}); got != nil {
|
||||
t.Errorf("its own names were replaced: %v", got)
|
||||
}
|
||||
if got := categoriesFor([]string{"llama-guard-3-1b"}, lg); got != nil {
|
||||
t.Errorf("Llama Guard's own codes were replaced: %v", got)
|
||||
}
|
||||
// Both kinds asked in turn: each must have something to count.
|
||||
got := categoriesFor([]string{"qwen3guard-4b", "llama-guard-3-1b"}, []string{"Violent"})
|
||||
if !slices.Contains(got, "Violent") || !slices.Contains(got, "S1") {
|
||||
t.Errorf("mixed: %v", got)
|
||||
}
|
||||
if got := categoriesFor([]string{"our-own-judge"}, lg); got != nil {
|
||||
t.Errorf("a judge llmgw does not know: %v", got)
|
||||
}
|
||||
if got := categoriesFor([]string{"qwen3guard-0.6b"}, nil); got != nil {
|
||||
t.Errorf("no categories (every one counts) became %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
// knownJudge is a judge model llmgw knows where to fetch and how to serve:
|
||||
// what a policy may name without anyone saying where its weights are.
|
||||
type knownJudge struct {
|
||||
Name, Repo, File, Format string
|
||||
// GPU is a judge too slow on CPUs to wait for: served on one card.
|
||||
GPU bool
|
||||
About string
|
||||
}
|
||||
|
||||
// knownJudges, the default first. Measured on the reference cluster against
|
||||
// ordinary coding requests ("continue", "kill the process on port 8080") and
|
||||
// harmful ones: Llama Guard 3 1B scored both alike, about one in two
|
||||
// ordinary ones refused; Qwen3Guard-Gen 0.6B kept them apart, 0.15 against
|
||||
// 1.00 at the worst, at about two seconds a check on two CPU threads.
|
||||
var knownJudges = []knownJudge{
|
||||
{Name: "qwen3guard-0.6b", Repo: "mradermacher/Qwen3Guard-Gen-0.6B-GGUF", File: "Qwen3Guard-Gen-0.6B.Q8_0.gguf",
|
||||
Format: "qwen3guard", About: "Qwen3Guard-Gen 0.6B on CPUs, 0.8 GB: the default"},
|
||||
{Name: "qwen3guard-4b", Repo: "mradermacher/Qwen3Guard-Gen-4B-GGUF", File: "Qwen3Guard-Gen-4B.Q4_K_M.gguf",
|
||||
Format: "qwen3guard", GPU: true, About: "Qwen3Guard-Gen 4B on a GPU, 2.5 GB"},
|
||||
{Name: "qwen3guard-8b", Repo: "mradermacher/Qwen3Guard-Gen-8B-GGUF", File: "Qwen3Guard-Gen-8B.Q4_K_M.gguf",
|
||||
Format: "qwen3guard", GPU: true, About: "Qwen3Guard-Gen 8B on a GPU, 4.7 GB"},
|
||||
{Name: "llama-guard-3-1b", Repo: "QuantFactory/Llama-Guard-3-1B-GGUF", File: "Llama-Guard-3-1B.Q4_K_M.gguf",
|
||||
Format: "llamaguard3", About: "Llama Guard 3 1B on CPUs, 0.9 GB: refuses ordinary coding requests"},
|
||||
{Name: "llama-guard-3-8b", Repo: "QuantFactory/Llama-Guard-3-8B-GGUF", File: "Llama-Guard-3-8B.Q4_K_M.gguf",
|
||||
Format: "llamaguard3", GPU: true, About: "Llama Guard 3 8B on a GPU, 4.6 GB"},
|
||||
}
|
||||
|
||||
const (
|
||||
defaultJudge = "qwen3guard-0.6b"
|
||||
defaultGPUJudge = "qwen3guard-4b"
|
||||
)
|
||||
|
||||
// baselineCategories are what the baseline policy counts of each format's
|
||||
// verdicts: not PII, which is the patterns', nor what refuses ordinary work
|
||||
// — copyright, unethical acts, politically sensitive topics; Llama Guard's
|
||||
// defamation, specialized advice, intellectual property and elections.
|
||||
var baselineCategories = map[string][]string{
|
||||
"qwen3guard": {"Violent", "Non-violent Illegal Acts", "Sexual Content or Sexual Acts", "Suicide & Self-Harm", "Jailbreak"},
|
||||
"llamaguard3": {"S1", "S2", "S3", "S4", "S7", "S9", "S10", "S11", "S12"},
|
||||
}
|
||||
|
||||
// reported is whether a judge of that format ever names category c: its own
|
||||
// names, or for Llama Guard its codes.
|
||||
func reported(format, c string) bool {
|
||||
if slices.ContainsFunc(guard.DefaultCategories(format), func(n string) bool { return strings.EqualFold(n, c) }) {
|
||||
return true
|
||||
}
|
||||
var n int
|
||||
_, err := fmt.Sscanf(c, "S%d", &n)
|
||||
return format == "llamaguard3" && err == nil && n >= 1 && n <= 14
|
||||
}
|
||||
|
||||
// categoriesFor is what a policy asking these judges should count, when the
|
||||
// categories it has are ones none of them names: a policy moved from Llama
|
||||
// Guard's S1 to Qwen3Guard, which says Violent, would otherwise count
|
||||
// nothing. Nil keeps what it has; so does a judge llmgw does not know.
|
||||
func categoriesFor(judges, have []string) []string {
|
||||
var formats []string
|
||||
for _, j := range judges {
|
||||
k, ok := knownJudgeNamed(j)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if !slices.Contains(formats, k.Format) {
|
||||
formats = append(formats, k.Format)
|
||||
}
|
||||
}
|
||||
if len(have) == 0 {
|
||||
return nil
|
||||
}
|
||||
stray := false
|
||||
for _, f := range formats {
|
||||
for _, c := range have {
|
||||
if !reported(f, c) && !slices.ContainsFunc(formats, func(o string) bool { return reported(o, c) }) {
|
||||
stray = true
|
||||
}
|
||||
}
|
||||
// Every judge must have something it counts.
|
||||
if !slices.ContainsFunc(have, func(c string) bool { return reported(f, c) }) {
|
||||
stray = true
|
||||
}
|
||||
}
|
||||
if !stray {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, f := range formats {
|
||||
out = append(out, baselineCategories[f]...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func knownJudgeNamed(name string) (knownJudge, bool) {
|
||||
i := slices.IndexFunc(knownJudges, func(k knownJudge) bool { return k.Name == name })
|
||||
if i < 0 {
|
||||
return knownJudge{}, false
|
||||
}
|
||||
return knownJudges[i], true
|
||||
}
|
||||
|
||||
func knownJudgeNames() string {
|
||||
var n []string
|
||||
for _, k := range knownJudges {
|
||||
n = append(n, k.Name)
|
||||
}
|
||||
return strings.Join(n, ", ")
|
||||
}
|
||||
|
||||
// judgeServeOptions are how a judge is served beyond which weights.
|
||||
type judgeServeOptions struct {
|
||||
Replicas, Cores int
|
||||
// GPUClass is the cards a GPU judge runs on; empty lets the plan choose.
|
||||
GPUClass string
|
||||
}
|
||||
|
||||
// serveRequest is the serve job for a judge. A judge on CPUs is critical,
|
||||
// as every judged request waits on it; one on a GPU is standard, so it takes
|
||||
// a card only from what may give one up, and the judge on CPUs after it in
|
||||
// the policies stands in while it has none.
|
||||
func (k knownJudge) serveRequest(o judgeServeOptions) modelserve.Request {
|
||||
req := modelserve.Request{
|
||||
Repo: k.Repo, File: k.File, Name: k.Name, GuardFormat: k.Format,
|
||||
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
|
||||
Replicas: o.Replicas, CtxPool: 8192, ContextLength: 4096,
|
||||
}
|
||||
if k.GPU {
|
||||
req.GPUClass, req.GPUsPerNode, req.Class = o.GPUClass, 1, catalog.ClassStandard
|
||||
if req.Replicas == 0 {
|
||||
req.Replicas = 1
|
||||
}
|
||||
return req
|
||||
}
|
||||
req.CPU, req.CPUCores = true, o.Cores
|
||||
if req.Replicas == 0 {
|
||||
req.Replicas = 2
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
func (k knownJudge) String() string {
|
||||
where := "CPUs"
|
||||
if k.GPU {
|
||||
where = "a GPU"
|
||||
}
|
||||
return fmt.Sprintf("%s (%s, %s/%s, on %s)", k.Name, k.Format, k.Repo, k.File, where)
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
func newKBCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
Use: "kb",
|
||||
Short: "Knowledge bases: collections of documents requests are answered from",
|
||||
Long: "A collection belongs to an organization, team, project or person, and is\n" +
|
||||
"embedded by one model of the catalog. People put documents into it from\n" +
|
||||
"their own machines with llmgw-kb (its own download, for Windows, macOS and\n" +
|
||||
"Linux), which sends text and vectors, never the files.\n\n" +
|
||||
"Who may read or add to a collection: its owner and everything below it,\n" +
|
||||
"and whoever it is granted to (grant), never outside its organization.\n" +
|
||||
"Targets are written org:Acme, team:Acme/hr, project:Acme/hr/docs (or with\n" +
|
||||
"IDs), key:KEY-ID or user:USER-ID. A collection is named by its ID, its name\n" +
|
||||
"when that is unique, or ORG/NAME.\n\n" +
|
||||
"Needs the PostgreSQL store with pgvector; `llmgw ctl kb list` says why not.",
|
||||
}
|
||||
c.AddCommand(&cobra.Command{
|
||||
Use: "list",
|
||||
Short: "Every collection: owner, model, documents and grants",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
var v adminapi.KBView
|
||||
if err := client().get(cmd.Context(), "/api/v1/kb", &v); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
if !v.Available {
|
||||
fmt.Fprintln(out, "Knowledge bases are off: "+v.Reason)
|
||||
return nil
|
||||
}
|
||||
if len(v.Collections) == 0 {
|
||||
fmt.Fprintln(out, "No collections yet (llmgw ctl kb create).")
|
||||
return nil
|
||||
}
|
||||
w := tab()
|
||||
fmt.Fprintln(w, "COLLECTION\tOWNER\tMODEL\tDOCUMENTS\tCHUNKS\tGRANTED TO")
|
||||
for _, c := range v.Collections {
|
||||
var grants []string
|
||||
for _, g := range c.Grants {
|
||||
grants = append(grants, or(g.Name, g.Target)+" ("+g.Perm+")")
|
||||
}
|
||||
fmt.Fprintf(w, "%s/%s\t%s\t%s (%d)\t%d\t%d\t%s\n", or(c.OrgName, c.Org), c.Name, or(c.OwnerName, c.Owner),
|
||||
c.EmbedModel, c.Dims, c.Documents, c.Chunks, strings.Join(grants, ", "))
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(v.Defaults) > 0 {
|
||||
fmt.Fprintln(out, "\nAnswered from by default, when a request names no collection:")
|
||||
w = tab()
|
||||
for _, d := range v.Defaults {
|
||||
fmt.Fprintf(w, " %s\t%s\n", or(d.Name, d.Target), strings.Join(d.Collections, ", "))
|
||||
}
|
||||
return w.Flush()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
var org string
|
||||
var dims int
|
||||
create := &cobra.Command{
|
||||
Use: "create OWNER NAME MODEL",
|
||||
Short: "Make a collection owned by an organization, team, project or person",
|
||||
Long: "MODEL is a catalog model with capabilities.embeddings. Its dimension is asked\n" +
|
||||
"of it, so it has to be able to serve now; --dims gives it instead.",
|
||||
Example: " llmgw ctl kb create team:Acme/hr policies bge-m3\n" +
|
||||
" llmgw ctl kb create user:USER-ID notes bge-m3 --org Acme",
|
||||
Args: cobra.ExactArgs(3),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
body, _ := json.Marshal(adminapi.KBCreateRequest{Owner: args[0], Name: args[1], Model: args[2], Org: org, Dims: dims})
|
||||
var v adminapi.KBCollectionView
|
||||
if err := client().do(cmd.Context(), "POST", "/api/v1/kb/collections", json.RawMessage(body), &v); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "collection %s (%s), embedded by %s in %d dimensions\n", v.Name, v.ID, v.EmbedModel, v.Dims)
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "people with access fill it with: llmgw-kb sync %s FOLDER\n", v.Name)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
create.Flags().StringVar(&org, "org", "", "the organization of a person's collection")
|
||||
create.Flags().IntVar(&dims, "dims", 0, "the model's embedding dimension, instead of asking it")
|
||||
c.AddCommand(create)
|
||||
|
||||
c.AddCommand(&cobra.Command{
|
||||
Use: "delete COLLECTION",
|
||||
Short: "Remove a collection, its documents and its grants",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if err := client().do(cmd.Context(), "DELETE", "/api/v1/kb/collections/"+url.PathEscape(args[0]), nil, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "collection %s removed\n", args[0])
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
for _, verb := range []struct {
|
||||
use, short, method string
|
||||
perm bool
|
||||
}{
|
||||
{"grant", "Let a scope and everything below it read a collection, or with --write add to it", "POST", true},
|
||||
{"revoke", "Take a grant away", "DELETE", false},
|
||||
} {
|
||||
var write bool
|
||||
cmd := &cobra.Command{
|
||||
Use: verb.use + " COLLECTION TARGET",
|
||||
Short: verb.short,
|
||||
Example: " llmgw ctl kb " + verb.use + " Acme/policies project:Acme/sales/crm",
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
req := adminapi.KBGrantRequest{Collection: args[0], Target: args[1], Perm: "read"}
|
||||
if write {
|
||||
req.Perm = "write"
|
||||
}
|
||||
body, _ := json.Marshal(req)
|
||||
if err := client().do(cmd.Context(), verb.method, "/api/v1/kb/grants", json.RawMessage(body), nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if verb.perm {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s may %s %s\n", args[1], req.Perm, args[0])
|
||||
} else {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s no longer granted %s\n", args[1], args[0])
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
if verb.perm {
|
||||
cmd.Flags().BoolVar(&write, "write", false, "add and remove documents too")
|
||||
}
|
||||
c.AddCommand(cmd)
|
||||
}
|
||||
|
||||
c.AddCommand(&cobra.Command{
|
||||
Use: "docs COLLECTION",
|
||||
Short: "A collection's documents",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var docs []adminapi.KBDocumentView
|
||||
if err := client().get(cmd.Context(), "/api/v1/kb/collections/"+url.PathEscape(args[0])+"/documents", &docs); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(docs)
|
||||
}
|
||||
w := tab()
|
||||
fmt.Fprintln(w, "DOCUMENT\tCHUNKS\tSIZE\tADDED\tBY")
|
||||
for _, d := range docs {
|
||||
fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\n", d.Name, d.Chunks, humanBytes(d.Bytes), d.AddedAt.Local().Format("2006-01-02 15:04"), d.AddedBy)
|
||||
}
|
||||
return w.Flush()
|
||||
},
|
||||
})
|
||||
c.AddCommand(&cobra.Command{
|
||||
Use: "rm COLLECTION DOCUMENT",
|
||||
Short: "Remove one document from a collection",
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
path := "/api/v1/kb/collections/" + url.PathEscape(args[0]) + "/documents?name=" + url.QueryEscape(args[1])
|
||||
if err := client().do(cmd.Context(), "DELETE", path, nil, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s removed from %s\n", args[1], args[0])
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
var clear bool
|
||||
def := &cobra.Command{
|
||||
Use: "default TARGET [COLLECTION...]",
|
||||
Short: "Answer a part of the tree's chat requests from collections, when they name none",
|
||||
Long: "Requests from TARGET and everything below it are then answered from these\n" +
|
||||
"collections without naming them (X-Llmgw-Knowledge). The narrowest default\n" +
|
||||
"wins: a key's over its project's, a project's over its team's. A request's own\n" +
|
||||
"header overrides it, and X-Llmgw-Knowledge: none turns it off. A collection a\n" +
|
||||
"caller may not read is left out for them, and a search that fails answers\n" +
|
||||
"without passages rather than refusing. In force within 30 seconds.",
|
||||
Example: " llmgw ctl kb default team:Acme/hr Acme/policies Acme/forms\n" +
|
||||
" llmgw ctl kb default team:Acme/hr --clear",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if clear != (len(args) == 1) {
|
||||
return fmt.Errorf("name collections, or --clear")
|
||||
}
|
||||
body, _ := json.Marshal(adminapi.KBDefaultRequest{Target: args[0], Collections: args[1:]})
|
||||
if err := client().do(cmd.Context(), "POST", "/api/v1/kb/defaults", json.RawMessage(body), nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if clear {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s answers from no collection by default\n", args[0])
|
||||
} else {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s answers from %s by default\n", args[0], strings.Join(args[1:], ", "))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
def.Flags().BoolVar(&clear, "clear", false, "no default any more")
|
||||
c.AddCommand(def)
|
||||
|
||||
var k int
|
||||
search := &cobra.Command{
|
||||
Use: "search COLLECTION QUERY...",
|
||||
Short: "Try a search of a collection, as an administrator (no tenancy applies)",
|
||||
Args: cobra.MinimumNArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
body, _ := json.Marshal(adminapi.KBSearchRequest{Collection: args[0], Query: strings.Join(args[1:], " "), K: k})
|
||||
var hits []adminapi.KBHitView
|
||||
if err := client().do(cmd.Context(), "POST", "/api/v1/kb/search", json.RawMessage(body), &hits); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(hits)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
if len(hits) == 0 {
|
||||
fmt.Fprintln(out, "Nothing found.")
|
||||
}
|
||||
for i, h := range hits {
|
||||
fmt.Fprintf(out, "%d. %s #%d\n%s\n\n", i+1, h.Document, h.Ord, indent(strings.TrimSpace(h.Text), " "))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
search.Flags().IntVarP(&k, "k", "k", 5, "how many chunks")
|
||||
c.AddCommand(search)
|
||||
return c
|
||||
}
|
||||
@@ -63,6 +63,10 @@ func Root() *cobra.Command {
|
||||
newWipeCmd(),
|
||||
newDocsCmd(),
|
||||
)
|
||||
// status, start, stop and restart: the whole system; service: this
|
||||
// machine's services.
|
||||
root.AddCommand(newSystemCmds()...)
|
||||
root.AddCommand(newServiceCmd())
|
||||
return root
|
||||
}
|
||||
|
||||
|
||||
+72
-1
@@ -5,6 +5,7 @@ import (
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
@@ -186,6 +187,7 @@ func newServerCmd() *cobra.Command {
|
||||
|
||||
rt.Start(ctx)
|
||||
defer rt.Stop()
|
||||
rt.SetStopped(cfg.Serving.Stopped)
|
||||
|
||||
// Which cluster this is, to federated peers: the id in the hop
|
||||
// header that stops a request going round two clusters that
|
||||
@@ -252,6 +254,8 @@ func newServerCmd() *cobra.Command {
|
||||
}
|
||||
defer ctl.Close()
|
||||
agentSvc = svc
|
||||
// Stopped by the operators (llmgw stop): nothing placed.
|
||||
ctl.Reconciler().SetStopped(cfg.Serving.Stopped)
|
||||
// Whether nodes keep llmgw's firewall, as the settings say now.
|
||||
svc.SetFirewall(func() *llmgwv1.Firewall {
|
||||
c := current.Load()
|
||||
@@ -361,12 +365,19 @@ func newServerCmd() *cobra.Command {
|
||||
evals = newEvalService(cfg, rt, admin, log)
|
||||
api.SetEval(evals)
|
||||
if st := rt.Store(); st != nil && gate != nil {
|
||||
api.SetGuard(&adminapi.GuardAdmin{Library: gate.Library, Store: st, Invalidate: gate.Invalidate})
|
||||
api.SetGuard(&adminapi.GuardAdmin{Library: gate.Library, Store: st, Invalidate: gate.Invalidate,
|
||||
JudgeErrors: gate.JudgeErrors, Ask: gate.AskJudge})
|
||||
}
|
||||
if st := rt.Store(); st != nil {
|
||||
// Removing a node revokes its certificate.
|
||||
api.SetRevoker(st.RevokeNode)
|
||||
if agentSvc != nil {
|
||||
api.SetAgentRestarter(agentSvc.RestartAgents)
|
||||
}
|
||||
api.SetAudit(st)
|
||||
// Knowledge bases, with the cluster's own models to embed a
|
||||
// probe (a collection's dimension) and an operator's search.
|
||||
api.SetKB(&adminapi.KBAdmin{Store: st, Embed: embedder(rt.Replay)})
|
||||
// The cluster's settings, each change checked against this
|
||||
// machine's own keys before it is kept.
|
||||
api.SetSettings(&adminapi.Settings{Store: st, LocalKeys: config.LocalKeys(),
|
||||
@@ -511,6 +522,21 @@ func newServerCmd() *cobra.Command {
|
||||
})
|
||||
go serve.WatchCanaries(ctx, 30*time.Second)
|
||||
adminAPI.SetServe(serve)
|
||||
if gate != nil {
|
||||
// A judge a policy names and the catalog lacks is
|
||||
// fetched and served, on the leader.
|
||||
keeper := &judgeKeeper{
|
||||
library: gate.Library,
|
||||
models: func() []catalog.Model { return admin.Catalog().Models() },
|
||||
serve: serve,
|
||||
leader: func() bool {
|
||||
return admin.Leadership() == nil || admin.Leadership().IsLeader()
|
||||
},
|
||||
gpus: func() []adminapi.ClassUse { return adminAPI.Summary().GPUs },
|
||||
log: log,
|
||||
}
|
||||
go keeper.run(ctx, 30*time.Second)
|
||||
}
|
||||
updateMetrics.Set(serve)
|
||||
go serve.WatchUpdates(ctx, cfg.Mirror.UpdateCheck)
|
||||
}
|
||||
@@ -675,6 +701,14 @@ func newServerCmd() *cobra.Command {
|
||||
return false
|
||||
}
|
||||
current.Store(&next)
|
||||
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "serving") }) {
|
||||
rt.SetStopped(next.Serving.Stopped)
|
||||
if admin != nil {
|
||||
admin.Reconciler().SetStopped(next.Serving.Stopped)
|
||||
}
|
||||
log.Warn("the service is " + map[bool]string{true: "stopped", false: "started"}[next.Serving.Stopped] +
|
||||
" by the cluster's settings")
|
||||
}
|
||||
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "firewall") }) {
|
||||
if err := writeFirewallState(next); err != nil {
|
||||
log.Warn("could not record the firewall's settings for this machine", "err", err)
|
||||
@@ -1153,3 +1187,40 @@ func federationID(ctx context.Context, st *storepkg.DB) (string, error) {
|
||||
// more of a secret than needed.
|
||||
return secret[:16], nil
|
||||
}
|
||||
|
||||
// embedder has a model embed one text through the gateway itself, the way a
|
||||
// kept request is replayed: for the administrator's knowledge-base
|
||||
// endpoints, not for anybody's traffic.
|
||||
func embedder(replay func(ctx context.Context, path, model string, body []byte) (int, []byte, time.Duration, error)) func(context.Context, string, string) ([]float32, error) {
|
||||
return func(ctx context.Context, model, text string) ([]float32, error) {
|
||||
body, _ := json.Marshal(map[string]any{"model": model, "input": []string{text}})
|
||||
status, out, _, err := replay(ctx, "/v1/embeddings", model, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if status != http.StatusOK {
|
||||
msg := strings.TrimSpace(string(out))
|
||||
var e struct {
|
||||
Error struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(out, &e) == nil && e.Error.Message != "" {
|
||||
msg = e.Error.Message
|
||||
}
|
||||
return nil, fmt.Errorf("%d: %s", status, msg)
|
||||
}
|
||||
var res struct {
|
||||
Data []struct {
|
||||
Embedding []float32 `json:"embedding"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(out, &res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(res.Data) != 1 || len(res.Data[0].Embedding) == 0 {
|
||||
return nil, fmt.Errorf("%s answered no embedding", model)
|
||||
}
|
||||
return res.Data[0].Embedding, nil
|
||||
}
|
||||
}
|
||||
@@ -293,7 +293,7 @@ func (in *serverInit) configJudgesWith(model string) bool {
|
||||
return false
|
||||
}
|
||||
for _, p := range cfg.Guard.Policies {
|
||||
if p.Judge != nil && p.Judge.Model == model {
|
||||
if slices.Contains(p.Judge.Judges(), model) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -308,18 +308,21 @@ func guardSection(judge bool) string {
|
||||
# judge: {model: ` + guardModelName + `, input: block, output: block, ...}
|
||||
`
|
||||
if judge {
|
||||
j = ` # A judge model reads what patterns cannot: violence, weapons, abuse,
|
||||
# self-harm. Two replicas run on CPUs with AVX2, about half a second
|
||||
# for a short text; an answer is read every 600 characters as it
|
||||
# streams. With fail: closed, a request the judge cannot read within
|
||||
j = ` # A judge model reads what patterns cannot: violence, crime, sexual
|
||||
# content, self-harm, jailbreaks. Two replicas run on CPUs, about two
|
||||
# seconds for a short text; an answer is read every 600 characters as
|
||||
# it streams. With fail: closed, a request no judge can read within
|
||||
# latency_budget is refused (503); open serves it, flagged, unjudged.
|
||||
# Defamation (S5), specialized advice (S6), intellectual property (S8)
|
||||
# and elections (S13) are left out: they refuse too much ordinary work.
|
||||
# PII is the patterns' (above), and copyright, unethical acts and
|
||||
# politically sensitive topics are left out: they refuse too much
|
||||
# ordinary work. Controversial counts as safe; controversial: unsafe
|
||||
# makes it count. With a GPU: llmgw ctl guard judge enable --gpu, then
|
||||
# llmgw ctl guard judge use ` + defaultGPUJudge + ` ` + guardModelName + `.
|
||||
judge:
|
||||
model: ` + guardModelName + `
|
||||
models: [` + guardModelName + `]
|
||||
input: block
|
||||
output: block
|
||||
categories: [S1, S2, S3, S4, S7, S9, S10, S11, S12]
|
||||
categories: [Violent, Non-violent Illegal Acts, Sexual Content or Sexual Acts, Suicide & Self-Harm, Jailbreak]
|
||||
latency_budget: 5s
|
||||
`
|
||||
}
|
||||
|
||||
@@ -41,11 +41,13 @@ const (
|
||||
starterSource = "hf:Qwen/Qwen2.5-0.5B-Instruct-GGUF/qwen2.5-0.5b-instruct-q4_k_m.gguf"
|
||||
starterVersion = "starter"
|
||||
|
||||
// The guardrails' judge: Meta's Llama Guard 3 1B, which reads a prompt
|
||||
// or an answer and says whether it is unsafe and why. Q4_K_M is about
|
||||
// 900 MB and judges a short text in about half a second on a CPU.
|
||||
guardModelName = "llama-guard-3-1b"
|
||||
guardModelSource = "hf:QuantFactory/Llama-Guard-3-1B-GGUF/Llama-Guard-3-1B.Q4_K_M.gguf"
|
||||
// The guardrails' judge: Qwen3Guard-Gen 0.6B, which reads a prompt or an
|
||||
// answer and says whether it is safe, controversial or unsafe, and why.
|
||||
// Q8_0 is 0.8 GB and judges a short text in about two seconds on two CPU
|
||||
// threads. It replaced Llama Guard 3 1B, which refused about one in two
|
||||
// ordinary coding requests ("continue").
|
||||
guardModelName = defaultJudge
|
||||
guardModelSource = "hf:mradermacher/Qwen3Guard-Gen-0.6B-GGUF/Qwen3Guard-Gen-0.6B.Q8_0.gguf"
|
||||
guardModelVersion = "guard"
|
||||
|
||||
// localLabel marks the control plane's own node, so the starter model is
|
||||
@@ -443,12 +445,6 @@ func starterModel(source string, gpuClasses []string, pinned bool) catalog.Model
|
||||
}
|
||||
}
|
||||
|
||||
// llamaGuard3Categories name Llama Guard 3's S1 to S13, so a verdict is
|
||||
// recorded as "Violent Crimes" rather than "S1".
|
||||
var llamaGuard3Categories = []string{"Violent Crimes", "Non-Violent Crimes", "Sex-Related Crimes",
|
||||
"Child Sexual Exploitation", "Defamation", "Specialized Advice", "Privacy", "Intellectual Property",
|
||||
"Indiscriminate Weapons", "Hate", "Suicide & Self-Harm", "Sexual Content", "Elections"}
|
||||
|
||||
// guardModel is the judge's catalog entry: CPU only, so it takes no card from
|
||||
// the models people use, on whichever node has room.
|
||||
func guardModel(source string) catalog.Model {
|
||||
@@ -457,7 +453,7 @@ func guardModel(source string) catalog.Model {
|
||||
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
|
||||
Source: source,
|
||||
Role: catalog.RoleGuard,
|
||||
Guard: &catalog.GuardModel{Format: "llamaguard3", Categories: llamaGuard3Categories},
|
||||
Guard: &catalog.GuardModel{Format: "qwen3guard"},
|
||||
// Two: every judged request waits on it, and with fail: closed a
|
||||
// judge that is not there refuses everything.
|
||||
Scaling: catalog.Scaling{Min: 2, Max: 2},
|
||||
|
||||
@@ -7,12 +7,14 @@ import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
func TestTheInitAgentUnitIsTheShippedUnit(t *testing.T) {
|
||||
@@ -150,9 +152,17 @@ func TestTheInstalledConfigurationIsHardenedAndLoads(t *testing.T) {
|
||||
if (p.Judge != nil) != judge {
|
||||
t.Errorf("judge=%v: the policy's judge is %+v", judge, p.Judge)
|
||||
}
|
||||
if judge && (p.Judge.Model != guardModelName || p.Judge.Input != "block") {
|
||||
if judge && (!slices.Equal(p.Judge.Judges(), []string{guardModelName}) || p.Judge.Input != "block") {
|
||||
t.Errorf("judge %+v", p.Judge)
|
||||
}
|
||||
// A category the judge never names would never count.
|
||||
if judge {
|
||||
for _, c := range p.Judge.Categories {
|
||||
if !slices.Contains(guard.DefaultCategories("qwen3guard"), c) {
|
||||
t.Errorf("the baseline counts %q, which Qwen3Guard does not name", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
if cfg.Auth.MaxKeyTTL != 90*24*time.Hour || cfg.Auth.SignupOpen || cfg.DebugPprof || cfg.AdminRemote {
|
||||
t.Errorf("not hardened: ttl %v signup %v pprof %v remote %v",
|
||||
cfg.Auth.MaxKeyTTL, cfg.Auth.SignupOpen, cfg.DebugPprof, cfg.AdminRemote)
|
||||
@@ -161,13 +171,13 @@ func TestTheInstalledConfigurationIsHardenedAndLoads(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTheGuardModelIsAValidJudge(t *testing.T) {
|
||||
m := guardModel("http://10.0.0.5:8081/llama-guard-3-1b/guard/g.gguf")
|
||||
m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf")
|
||||
c, err := catalog.New([]catalog.Model{m})
|
||||
if err != nil {
|
||||
t.Fatalf("not a valid catalog entry: %v", err)
|
||||
}
|
||||
got, _ := c.Resolve(guardModelName)
|
||||
if !got.IsGuard() || got.Guard.Format != "llamaguard3" || len(got.Guard.Categories) != 13 || got.Scaling.Min != 2 {
|
||||
if !got.IsGuard() || got.Guard.Format != "qwen3guard" || got.Scaling.Min != 2 {
|
||||
t.Errorf("%+v", got)
|
||||
}
|
||||
for _, p := range got.Profiles {
|
||||
|
||||
@@ -370,6 +370,8 @@ func (in *serverInit) setupPostgres(ctx context.Context) (string, error) {
|
||||
in.done("PostgreSQL sized for a machine it shares (shared_buffers 96MB, 40 connections)")
|
||||
}
|
||||
|
||||
in.setupPgvector(ctx, discovery.DiscoverHost())
|
||||
|
||||
// Proof the address works, before anything is moved into it.
|
||||
address := "@" + in.dsnPath()
|
||||
s, err := store.Open(ctx, address)
|
||||
@@ -482,8 +484,17 @@ func lastLines(b []byte, n int) string {
|
||||
|
||||
// psql runs one statement as the postgres superuser, over the local socket.
|
||||
func (in *serverInit) psql(ctx context.Context, query string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, "runuser", "-u", "postgres", "--",
|
||||
in.pg.tool("psql"), "-v", "ON_ERROR_STOP=1", "-qAtX", "-c", query)
|
||||
return in.psqlIn(ctx, "", query)
|
||||
}
|
||||
|
||||
// psqlIn runs a statement as postgres in one database, or in postgres's
|
||||
// default one when db is "".
|
||||
func (in *serverInit) psqlIn(ctx context.Context, db, query string) (string, error) {
|
||||
args := []string{"-v", "ON_ERROR_STOP=1", "-qAtX", "-c", query}
|
||||
if db != "" {
|
||||
args = append(args, "-d", db)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "runuser", append([]string{"-u", "postgres", "--", in.pg.tool("psql")}, args...)...)
|
||||
// postgres cannot enter root's home, and psql says so on every call.
|
||||
cmd.Dir = "/"
|
||||
out, err := cmd.CombinedOutput()
|
||||
@@ -538,3 +549,50 @@ func (in *serverInit) stopService(ctx context.Context) (bool, error) {
|
||||
in.done("llmgw-server stopped while its store moves")
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// pgvectorPackages are the package names of pgvector for a PostgreSQL major
|
||||
// version: Debian's and Ubuntu's (and PGDG's apt repository) name it after the
|
||||
// server, PGDG's rpm repository after the version alone.
|
||||
func pgvectorPackages(major int) map[string][]string {
|
||||
return map[string][]string{
|
||||
"deb": {fmt.Sprintf("postgresql-%d-pgvector", major)},
|
||||
"rpm": {fmt.Sprintf("pgvector_%d", major)},
|
||||
}
|
||||
}
|
||||
|
||||
// setupPgvector installs pgvector and enables it in llmgw's database, for
|
||||
// knowledge bases. Not a reason to stop: without it everything else works,
|
||||
// and the store says why knowledge bases are off when somebody asks for one.
|
||||
func (in *serverInit) setupPgvector(ctx context.Context, h discovery.Host) {
|
||||
warn := func(format string, a ...any) {
|
||||
fmt.Fprintf(in.out, " !! pgvector: "+format+"; knowledge bases stay off\n", a...)
|
||||
}
|
||||
if on, err := in.psqlIn(ctx, pgDatabase, "SELECT 1 FROM pg_extension WHERE extname = 'vector'"); err == nil && on == "1" {
|
||||
in.kept("pgvector in database %s", pgDatabase)
|
||||
return
|
||||
}
|
||||
have, err := in.psqlBool(ctx, "SELECT 1 FROM pg_available_extensions WHERE name = 'vector'")
|
||||
if err != nil {
|
||||
warn("%v", err)
|
||||
return
|
||||
}
|
||||
if !have {
|
||||
num, err := in.psql(ctx, "SHOW server_version_num")
|
||||
n, convErr := strconv.Atoi(num)
|
||||
if err != nil || convErr != nil || h.Family() == "" {
|
||||
warn("cannot tell which package to install (version %q, %s)", num, h.OSID)
|
||||
return
|
||||
}
|
||||
pkgs := pgvectorPackages(n / 10000)
|
||||
pi := &provision.Installer{Host: h, Log: slog.Default()}
|
||||
if err := pi.InstallPackages(ctx, pkgs); err != nil {
|
||||
warn("installing %s: %v", strings.Join(pkgs[h.Family()], " "), err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if _, err := in.psqlIn(ctx, pgDatabase, "CREATE EXTENSION IF NOT EXISTS vector"); err != nil {
|
||||
warn("%v", err)
|
||||
return
|
||||
}
|
||||
in.done("pgvector enabled in database %s", pgDatabase)
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// llmgwServices are the units llmgw runs as on a machine, in the order they
|
||||
// start: the store's clients after the store, the server before the agent
|
||||
// that dials it.
|
||||
var llmgwServices = []string{"llmgw-mirror", "llmgw-server", "llmgw-agent"}
|
||||
|
||||
// unitDir is where llmgw's units are installed.
|
||||
var unitDir = "/etc/systemd/system"
|
||||
|
||||
// serviceRunner runs systemctl; a variable so tests stand in for it.
|
||||
var serviceRunner = func(ctx context.Context, args ...string) (string, error) {
|
||||
out, err := exec.CommandContext(ctx, "systemctl", args...).CombinedOutput() //nolint:gosec // fixed command
|
||||
return strings.TrimSpace(string(out)), err
|
||||
}
|
||||
|
||||
// installedServices are the llmgw units this machine has, or those named.
|
||||
func installedServices(ctx context.Context, names []string) ([]string, error) {
|
||||
var want []string
|
||||
for _, n := range names {
|
||||
if !strings.HasPrefix(n, "llmgw-") {
|
||||
n = "llmgw-" + n
|
||||
}
|
||||
want = append(want, n)
|
||||
}
|
||||
var have []string
|
||||
for _, u := range llmgwServices {
|
||||
if _, err := os.Stat(unitDir + "/" + u + ".service"); err != nil {
|
||||
continue
|
||||
}
|
||||
if len(want) > 0 && !hasUnit(want, u) {
|
||||
continue
|
||||
}
|
||||
have = append(have, u)
|
||||
}
|
||||
for _, w := range want {
|
||||
if !hasUnit(have, w) {
|
||||
return nil, fmt.Errorf("%s is not installed here (this machine has: %s)", w, strings.Join(present(), ", "))
|
||||
}
|
||||
}
|
||||
if len(have) == 0 {
|
||||
return nil, errors.New("no llmgw service is installed on this machine")
|
||||
}
|
||||
return have, nil
|
||||
}
|
||||
|
||||
func present() []string {
|
||||
var out []string
|
||||
for _, u := range llmgwServices {
|
||||
if _, err := os.Stat(unitDir + "/" + u + ".service"); err == nil {
|
||||
out = append(out, strings.TrimPrefix(u, "llmgw-"))
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return []string{"none"}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func hasUnit(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// newServiceCmd is llmgw service: this machine's llmgw services, one by one.
|
||||
// The whole system is llmgw status, start, stop and restart.
|
||||
func newServiceCmd() *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
Use: "service",
|
||||
Short: "This machine's llmgw services: status, start, stop, restart",
|
||||
Long: "The control plane (server), node agent (agent) and model mirror (mirror) on\n" +
|
||||
"this machine alone, by systemd. The whole system is 'llmgw status', 'llmgw\n" +
|
||||
"stop', 'llmgw start' and 'llmgw restart', run on a control plane.",
|
||||
}
|
||||
c.AddCommand(serviceVerbs()...)
|
||||
return c
|
||||
}
|
||||
|
||||
func serviceVerbs() []*cobra.Command {
|
||||
status := &cobra.Command{
|
||||
Use: "status [server|agent|mirror]...",
|
||||
Short: "Whether llmgw's services on this machine run",
|
||||
Long: "Each llmgw service this machine has — the control plane (server), the\n" +
|
||||
"node agent (agent), the model mirror (mirror) — and whether it runs.",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
units, err := localUnits(cmd, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
for _, u := range units {
|
||||
state, _ := serviceRunner(cmd.Context(), "is-active", u)
|
||||
since, _ := serviceRunner(cmd.Context(), "show", u, "-p", "ActiveEnterTimestamp", "--value")
|
||||
fmt.Fprintf(out, "%-14s %-10s %s\n", strings.TrimPrefix(u, "llmgw-"), state, since)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
verb := func(use, short, long, action string, reverse bool) *cobra.Command {
|
||||
var engines bool
|
||||
c := &cobra.Command{
|
||||
Use: use + " [server|agent|mirror]...",
|
||||
Short: short,
|
||||
Long: long,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if os.Geteuid() != 0 {
|
||||
return errors.New("starting and stopping services needs root: run it with sudo")
|
||||
}
|
||||
units, err := localUnits(cmd, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if reverse {
|
||||
// The agent before the server it dials, the server before
|
||||
// the mirror it reads.
|
||||
for i, j := 0, len(units)-1; i < j; i, j = i+1, j-1 {
|
||||
units[i], units[j] = units[j], units[i]
|
||||
}
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
for _, u := range units {
|
||||
if msg, err := serviceRunner(cmd.Context(), action, u); err != nil {
|
||||
return fmt.Errorf("%s %s: %v %s", action, u, err, msg)
|
||||
}
|
||||
state, _ := serviceRunner(cmd.Context(), "is-active", u)
|
||||
fmt.Fprintf(out, "%-14s %s\n", strings.TrimPrefix(u, "llmgw-"), state)
|
||||
}
|
||||
if action == "stop" && hasUnit(units, "llmgw-agent") {
|
||||
if engines {
|
||||
_, _ = exec.CommandContext(cmd.Context(), "pkill", "-u", "llmgw-engine").CombinedOutput() //nolint:gosec // fixed command
|
||||
fmt.Fprintln(out, "engines stopped (every process of llmgw-engine)")
|
||||
} else {
|
||||
fmt.Fprintln(out, "\nThe engines keep serving: they outlive the agent, which takes them over again when it starts.\n"+
|
||||
"--engines stops them too; the control plane then places their replicas elsewhere.")
|
||||
}
|
||||
}
|
||||
if action == "stop" && hasUnit(units, "llmgw-server") {
|
||||
fmt.Fprintln(out, "A standby control plane, where there is one, takes over within seconds.")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
if action == "stop" {
|
||||
c.Flags().BoolVar(&engines, "engines", false, "also stop the engines this node runs")
|
||||
}
|
||||
return c
|
||||
}
|
||||
return []*cobra.Command{
|
||||
status,
|
||||
verb("start", "Start llmgw's services on this machine",
|
||||
"Starts each llmgw service this machine has, or those named: the mirror, the\n"+
|
||||
"control plane, the node agent, in that order.", "start", false),
|
||||
verb("stop", "Stop llmgw's services on this machine",
|
||||
"Stops each llmgw service this machine has, or those named: the agent, the\n"+
|
||||
"control plane, the mirror, in that order. The engines a node runs keep\n"+
|
||||
"serving unless --engines is given; a standby control plane takes over from\n"+
|
||||
"a leader stopped. They stay stopped until 'llmgw start', or the machine\n"+
|
||||
"boots.", "stop", true),
|
||||
verb("restart", "Restart llmgw's services on this machine",
|
||||
"Restarts each llmgw service this machine has, or those named. Engines keep\n"+
|
||||
"serving and are taken over again by the agent; a control plane's settings\n"+
|
||||
"and server.yaml are read anew.", "restart", false),
|
||||
}
|
||||
}
|
||||
|
||||
func localUnits(cmd *cobra.Command, args []string) ([]string, error) {
|
||||
if runtime.GOOS != "linux" {
|
||||
return nil, fmt.Errorf("llmgw's services run under systemd on Linux; this is %s", runtime.GOOS)
|
||||
}
|
||||
return installedServices(cmd.Context(), args)
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
func TestTheServicesHereAreFoundAndAskedAbout(t *testing.T) {
|
||||
old, oldRun := unitDir, serviceRunner
|
||||
unitDir = t.TempDir()
|
||||
t.Cleanup(func() { unitDir, serviceRunner = old, oldRun })
|
||||
for _, u := range []string{"llmgw-server", "llmgw-agent"} {
|
||||
_ = os.WriteFile(filepath.Join(unitDir, u+".service"), []byte("[Unit]"), 0o644)
|
||||
}
|
||||
var asked []string
|
||||
serviceRunner = func(_ context.Context, args ...string) (string, error) {
|
||||
asked = append(asked, strings.Join(args, " "))
|
||||
if args[0] == "is-active" {
|
||||
return "active", nil
|
||||
}
|
||||
return "Wed 2026-10-07 12:00:00 UTC", nil
|
||||
}
|
||||
|
||||
units, err := installedServices(context.Background(), nil)
|
||||
if err != nil || strings.Join(units, ",") != "llmgw-server,llmgw-agent" {
|
||||
t.Fatalf("%v %v", units, err)
|
||||
}
|
||||
if _, err := installedServices(context.Background(), []string{"mirror"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "server, agent") {
|
||||
t.Errorf("a service not here: %v", err)
|
||||
}
|
||||
if u, err := installedServices(context.Background(), []string{"agent"}); err != nil || len(u) != 1 {
|
||||
t.Errorf("one named: %v %v", u, err)
|
||||
}
|
||||
|
||||
if runtime.GOOS != "linux" {
|
||||
return
|
||||
}
|
||||
var out bytes.Buffer
|
||||
c := serviceVerbs()[0]
|
||||
c.SetOut(&out)
|
||||
c.SetContext(context.Background())
|
||||
if err := c.RunE(c, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "server") || !strings.Contains(out.String(), "active") {
|
||||
t.Errorf("%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopSetsTheClustersServingStopped(t *testing.T) {
|
||||
var put adminapi.SettingsRequest
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/api/v1/summary":
|
||||
_ = json.NewEncoder(w).Encode(adminapi.Summary{})
|
||||
case r.Method == "GET" && r.URL.Path == "/api/v1/settings":
|
||||
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 3, Body: "router:\n retries: 2\n"})
|
||||
case r.Method == "PUT" && r.URL.Path == "/api/v1/settings":
|
||||
_ = json.NewDecoder(r.Body).Decode(&put)
|
||||
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 4, Body: put.Body})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
dir := t.TempDir()
|
||||
cfg := filepath.Join(dir, "server.yaml")
|
||||
_ = os.WriteFile(filepath.Join(dir, "token"), []byte("t"), 0o600)
|
||||
_ = os.WriteFile(cfg, []byte("listen: \":8080\"\nagent_insecure: true\nadmin_listen: "+
|
||||
strings.TrimPrefix(srv.URL, "http://")+"\nadmin_remote: true\nadmin_token_file: "+filepath.Join(dir, "token")+"\n"), 0o600)
|
||||
|
||||
var stop *cobra.Command
|
||||
for _, c := range newSystemCmds() {
|
||||
if c.Name() == "stop" {
|
||||
stop = c
|
||||
}
|
||||
}
|
||||
var out bytes.Buffer
|
||||
stop.SetOut(&out)
|
||||
stop.SetArgs([]string{"--config", cfg, "--note", "maintenance"})
|
||||
if err := stop.ExecuteContext(context.Background()); err != nil {
|
||||
t.Fatalf("%v: %s", err, out.String())
|
||||
}
|
||||
if put.Base != 3 || !strings.Contains(put.Body, "stopped: true") || !strings.Contains(put.Body, "retries: 2") ||
|
||||
put.Note != "maintenance" {
|
||||
t.Errorf("sent %+v", put)
|
||||
}
|
||||
if !strings.Contains(out.String(), "stopping") {
|
||||
t.Errorf("%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Started while the control plane has not yet taken the setting up: every
|
||||
// model of a stopped system has the nothing it asks for, which is not
|
||||
// serving.
|
||||
func TestStartWaitsForTheControlPlaneToServe(t *testing.T) {
|
||||
old := servicePoll
|
||||
servicePoll = time.Millisecond
|
||||
t.Cleanup(func() { servicePoll = old })
|
||||
answers := []adminapi.Summary{
|
||||
{Stopped: true, Models: adminapi.ModelTotals{Total: 2, Satisfied: 2}},
|
||||
{Models: adminapi.ModelTotals{Total: 2, Satisfied: 0}},
|
||||
{Models: adminapi.ModelTotals{Total: 2, Satisfied: 2}},
|
||||
}
|
||||
asked := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(answers[min(asked, len(answers)-1)])
|
||||
asked++
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
var out bytes.Buffer
|
||||
cmd := &cobra.Command{}
|
||||
cmd.SetOut(&out)
|
||||
cmd.SetContext(context.Background())
|
||||
c := &ctlClient{base: srv.URL, http: srv.Client()}
|
||||
if err := waitForService(cmd, func() *ctlClient { return c }, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked != 3 {
|
||||
t.Errorf("returned after %d answers: %s", asked, out.String())
|
||||
}
|
||||
}
|
||||
@@ -153,6 +153,19 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) {
|
||||
fmt.Fprintf(out, "STORE %s\n", strings.Join(parts, ", "))
|
||||
}
|
||||
|
||||
if s.Stopped {
|
||||
fmt.Fprintln(out, "SERVICE STOPPED: no model runs and requests are answered 503 (llmgw start)")
|
||||
}
|
||||
if s.GuardOff {
|
||||
fmt.Fprintln(out, "GUARD off: no request is checked, whatever the policies say (llmgw ctl guard on)")
|
||||
}
|
||||
if len(s.Judges) > 0 {
|
||||
fmt.Fprintln(out, "\nGUARDRAILS' JUDGE")
|
||||
for _, j := range s.Judges {
|
||||
fmt.Fprintf(out, " %s: %s\n", j.Model, j.Problem)
|
||||
}
|
||||
}
|
||||
|
||||
if len(s.Parked) > 0 {
|
||||
fmt.Fprintln(out, "\nPARKED — failed every time they started; not retried until fixed")
|
||||
for _, p := range s.Parked {
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
)
|
||||
|
||||
// systemClient is the admin API of the control plane on this machine, as its
|
||||
// server.yaml says, with its token.
|
||||
func systemClient(configPath string) func() *ctlClient {
|
||||
return func() *ctlClient {
|
||||
addr := "127.0.0.1:9090"
|
||||
var tokens []string
|
||||
if cfg, ok := loadServerConfig(configPath); ok {
|
||||
if cfg.AdminListen != "" {
|
||||
addr = cfg.AdminListen
|
||||
if strings.HasPrefix(addr, ":") {
|
||||
addr = "127.0.0.1" + addr
|
||||
}
|
||||
}
|
||||
tokens = append(tokens, adminTokenPath(configPath, cfg))
|
||||
}
|
||||
return newCtlClient(addr, tokens...)
|
||||
}
|
||||
}
|
||||
|
||||
// newSystemCmds are llmgw status, stop, start and restart: the whole system,
|
||||
// from a control plane.
|
||||
func newSystemCmds() []*cobra.Command {
|
||||
var configPath string
|
||||
client := systemClient("/etc/llmgw/server.yaml")
|
||||
bind := func(c *cobra.Command) *cobra.Command {
|
||||
c.Flags().StringVar(&configPath, "config", "/etc/llmgw/server.yaml", "this control plane's configuration")
|
||||
c.PreRun = func(*cobra.Command, []string) { client = systemClient(configPath) }
|
||||
return c
|
||||
}
|
||||
call := func() *ctlClient { return client() }
|
||||
|
||||
status := bind(&cobra.Command{
|
||||
Use: "status",
|
||||
Short: "The whole system: whether it serves, its control planes, nodes, models and store",
|
||||
Long: "Run on a control plane. Says whether the service is stopped (llmgw stop),\n" +
|
||||
"and everything 'llmgw ctl status' does. One machine's services are\n" +
|
||||
"'llmgw service status'.",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
var s adminapi.Summary
|
||||
if err := call().get(cmd.Context(), "/api/v1/summary", &s); err != nil {
|
||||
return notAControlPlane(err)
|
||||
}
|
||||
printSummary(cmd, s)
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
switchService := func(stop bool) *cobra.Command {
|
||||
use, short, long := "start", "Start the whole system serving again, after llmgw stop",
|
||||
"Every model comes back as its catalog entry asks: placed, loaded and\n"+
|
||||
"serving again, on every control plane within seconds. Run on a control\n"+
|
||||
"plane. The services themselves are never stopped by llmgw stop; a\n"+
|
||||
"machine's own are started with 'llmgw service start'."
|
||||
if stop {
|
||||
use, short, long = "stop", "Stop the whole system serving: every model taken down, requests answered 503",
|
||||
"Every model's replicas are taken down, freeing every GPU, and every\n"+
|
||||
"request for a model is answered 503 (system_stopped) until 'llmgw start'.\n"+
|
||||
"The control planes, agents and mirror keep running, so the console, the\n"+
|
||||
"admin pages and llmgw start still work, and no model's definition is\n"+
|
||||
"changed. In force on every control plane within seconds; kept in the\n"+
|
||||
"cluster's settings (serving.stopped), so a restart does not undo it."
|
||||
}
|
||||
var note string
|
||||
var wait bool
|
||||
c := bind(&cobra.Command{
|
||||
Use: use,
|
||||
Short: short,
|
||||
Long: long,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if note == "" {
|
||||
note = "llmgw " + use
|
||||
}
|
||||
var s adminapi.Summary
|
||||
if err := call().get(cmd.Context(), "/api/v1/summary", &s); err != nil {
|
||||
return notAControlPlane(err)
|
||||
}
|
||||
if s.Stopped == stop {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "the system is already %s\n", map[bool]string{true: "stopped", false: "serving"}[stop])
|
||||
return nil
|
||||
}
|
||||
if err := changeSettings(cmd, call, note, func(doc *settings.Doc) error {
|
||||
return doc.SetYAML([]string{"serving", "stopped"}, fmt.Sprint(stop))
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
if !wait {
|
||||
if stop {
|
||||
fmt.Fprintln(out, "stopping: every model's replicas are being taken down; llmgw status follows it")
|
||||
} else {
|
||||
fmt.Fprintln(out, "starting: every model is placed again as it asks; llmgw status follows it")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return waitForService(cmd, call, stop)
|
||||
},
|
||||
})
|
||||
c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version and in the audit")
|
||||
c.Flags().BoolVar(&wait, "wait", false, "wait until every replica is down (stop) or the models have theirs (start)")
|
||||
return c
|
||||
}
|
||||
|
||||
var sshUser string
|
||||
restart := bind(&cobra.Command{
|
||||
Use: "restart",
|
||||
Short: "Restart every control plane and node agent; the engines keep serving",
|
||||
Long: "Every node's agent starts again, then the other control planes over ssh\n" +
|
||||
"as --ssh-user, then this one — the leader last, so it hands over to a\n" +
|
||||
"standby already restarted. The engines keep serving throughout and are\n" +
|
||||
"taken over again; nothing is placed anew. Run on a control plane, with\n" +
|
||||
"sudo. One machine's services are 'llmgw service restart'.",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if os.Geteuid() != 0 {
|
||||
return errors.New("restarting this control plane needs root: run it with sudo")
|
||||
}
|
||||
ctx, out := cmd.Context(), cmd.OutOrStdout()
|
||||
var s adminapi.Summary
|
||||
if err := call().get(ctx, "/api/v1/summary", &s); err != nil {
|
||||
return notAControlPlane(err)
|
||||
}
|
||||
var agents adminapi.AgentRestartView
|
||||
if err := call().post(ctx, "/api/v1/agents/restart", []byte("{}"), &agents); err != nil {
|
||||
return fmt.Errorf("restarting the agents: %w", err)
|
||||
}
|
||||
fmt.Fprintf(out, "agents %d told to restart: %s\n", len(agents.Nodes), strings.Join(agents.Nodes, ", "))
|
||||
|
||||
// The other control planes, standbys before a leader.
|
||||
var others []adminapi.ControlPlane
|
||||
for _, cp := range s.ControlPlanes.Members {
|
||||
if !cp.This {
|
||||
others = append(others, cp)
|
||||
}
|
||||
}
|
||||
// One that cannot be reached is said and left: the rest of the
|
||||
// system still restarts, and the leader still goes last.
|
||||
var failed []string
|
||||
for pass := 0; pass < 2; pass++ {
|
||||
for _, cp := range others {
|
||||
leading := cp.State == "leading"
|
||||
if leading != (pass == 1) {
|
||||
continue
|
||||
}
|
||||
r := &onboard.SSH{Addr: cp.Address, User: sshUser}
|
||||
if res, err := r.Run(ctx, "sudo systemctl restart llmgw-server", nil); err != nil {
|
||||
fmt.Fprintf(out, "%-10s NOT restarted: ssh as %q: %v %s\n", cp.Name, sshUser, err, strings.TrimSpace(string(res)))
|
||||
failed = append(failed, cp.Name+" ("+cp.Address+")")
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(out, "%-10s restarted (%s)\n", cp.Name, cp.State)
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(unitDir + "/llmgw-server.service"); err == nil {
|
||||
fmt.Fprintln(out, "this restarting")
|
||||
if b, err := exec.CommandContext(ctx, "systemctl", "restart", "llmgw-server").CombinedOutput(); err != nil { //nolint:gosec // fixed command
|
||||
return fmt.Errorf("restarting this control plane: %v: %s", err, strings.TrimSpace(string(b)))
|
||||
}
|
||||
fmt.Fprintln(out, "this restarted")
|
||||
}
|
||||
fmt.Fprintln(out, "\nllmgw status shows the nodes coming back; their replicas are taken over as they are.")
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("not restarted: %s; run `sudo llmgw service restart server` there, "+
|
||||
"or give --ssh-user someone this machine's root can log in as", strings.Join(failed, ", "))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
restart.Flags().StringVar(&sshUser, "ssh-user", os.Getenv("SUDO_USER"), "who to log in to the other control planes as")
|
||||
|
||||
return []*cobra.Command{status, switchService(true), switchService(false), restart}
|
||||
}
|
||||
|
||||
// servicePoll is how often --wait asks; a variable so tests need not wait.
|
||||
var servicePoll = 5 * time.Second
|
||||
|
||||
// waitForService waits until no replica runs (stopped) or every model has
|
||||
// the replicas it asks for (started).
|
||||
//
|
||||
// Only once the control plane says it is in the state asked for: it takes the
|
||||
// setting up at its next look at the store, seconds later, and until then a
|
||||
// stopped system's models, which ask for nothing, all have what they ask for —
|
||||
// so llmgw start --wait returned at once, with nothing started.
|
||||
func waitForService(cmd *cobra.Command, call func() *ctlClient, stop bool) error {
|
||||
deadline := time.Now().Add(15 * time.Minute)
|
||||
for time.Now().Before(deadline) {
|
||||
var s adminapi.Summary
|
||||
if err := call().get(cmd.Context(), "/api/v1/summary", &s); err == nil && s.Stopped == stop {
|
||||
if stop && s.Replicas.Ready+s.Replicas.Starting+s.Replicas.Draining == 0 {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "stopped: no replica runs")
|
||||
return nil
|
||||
}
|
||||
if !stop && s.Models.Satisfied >= s.Models.Total {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "serving: %d of %d models have the replicas they ask for\n",
|
||||
s.Models.Satisfied, s.Models.Total)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-cmd.Context().Done():
|
||||
return cmd.Context().Err()
|
||||
case <-time.After(servicePoll):
|
||||
}
|
||||
}
|
||||
return errors.New("not there within fifteen minutes; llmgw status says where it stands")
|
||||
}
|
||||
|
||||
// notAControlPlane is a failure to reach the admin API, said as what it
|
||||
// most often means.
|
||||
func notAControlPlane(err error) error {
|
||||
return fmt.Errorf("%w\n(the whole system is stopped and started from a control plane; this machine's own "+
|
||||
"services are 'llmgw service status|start|stop|restart')", err)
|
||||
}
|
||||
@@ -34,10 +34,11 @@ import (
|
||||
// wrapper — left it in the content. A differently trained model fixes that. A
|
||||
// bigger one from the same family does not: qwen2.5-coder-7b invented
|
||||
// <function-call> and failed the same way.
|
||||
func newToolCheckCmd() *cobra.Command {
|
||||
func newToolCheckCmd(client func() *ctlClient) *cobra.Command {
|
||||
var (
|
||||
endpoint string
|
||||
model string
|
||||
key string
|
||||
timeout time.Duration
|
||||
)
|
||||
|
||||
@@ -57,10 +58,21 @@ func newToolCheckCmd() *cobra.Command {
|
||||
if endpoint == "" {
|
||||
return fmt.Errorf("--endpoint is required, e.g. http://10.0.0.145:20002")
|
||||
}
|
||||
// A replica's engine demands its own key, and refused the check
|
||||
// with a 401 when none was sent. Asked of the admin API, as bench
|
||||
// does; an endpoint that is no replica's is asked without one.
|
||||
if key == "" {
|
||||
reps, err := listReplicas(cmd, client)
|
||||
if err != nil {
|
||||
cmd.PrintErrf("The admin API could not be asked for the engine key of the replica at "+
|
||||
"--endpoint (%v); an engine that demands one will refuse. Pass --key to give it.\n", err)
|
||||
}
|
||||
key = replicaKey(reps, endpoint)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), timeout)
|
||||
defer cancel()
|
||||
|
||||
res, err := probeToolCall(ctx, endpoint, model)
|
||||
res, err := probeToolCall(ctx, endpoint, model, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -75,6 +87,7 @@ func newToolCheckCmd() *cobra.Command {
|
||||
}
|
||||
cmd.Flags().StringVar(&endpoint, "endpoint", "", "engine base URL, e.g. http://10.0.0.145:20002")
|
||||
cmd.Flags().StringVar(&model, "model", "", "model name to send, when the engine needs one")
|
||||
cmd.Flags().StringVar(&key, "key", "", "the engine's key (default: the admin API's, for the replica at --endpoint)")
|
||||
cmd.Flags().DurationVar(&timeout, "timeout", 5*time.Minute, "how long to wait; CPU inference is slow")
|
||||
return cmd
|
||||
}
|
||||
@@ -102,7 +115,7 @@ type ToolProbe struct {
|
||||
Raw string
|
||||
}
|
||||
|
||||
func probeToolCall(ctx context.Context, endpoint, model string) (ToolProbe, error) {
|
||||
func probeToolCall(ctx context.Context, endpoint, model, key string) (ToolProbe, error) {
|
||||
// A question with one obvious tool answer and no other sensible reply, so a
|
||||
// model that can use tools has no excuse not to.
|
||||
body := map[string]any{
|
||||
@@ -138,6 +151,9 @@ func probeToolCall(ctx context.Context, endpoint, model string) (ToolProbe, erro
|
||||
return ToolProbe{}, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if key != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+key)
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
@@ -150,6 +166,10 @@ func probeToolCall(ctx context.Context, endpoint, model string) (ToolProbe, erro
|
||||
return ToolProbe{}, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
if resp.StatusCode == http.StatusUnauthorized {
|
||||
return ToolProbe{}, fmt.Errorf("%s answered 401: the engine's key is the admin API's, "+
|
||||
"for the replica `llmgw ctl replicas` lists at --endpoint; or pass it with --key", url)
|
||||
}
|
||||
return ToolProbe{}, fmt.Errorf("%s returned %d: %s", url, resp.StatusCode, trim(string(raw), 300))
|
||||
}
|
||||
return readToolProbe(raw)
|
||||
|
||||
@@ -1,10 +1,65 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// toolCheck runs the command against an engine that answers only its own key,
|
||||
// with an admin API that lists the replica there with that key.
|
||||
func toolCheck(t *testing.T, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer engine-key" {
|
||||
http.Error(w, `{"error":{"message":"Invalid API Key"}}`, http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
fmt.Fprint(w, `{"choices":[{"message":{"tool_calls":[{"id":"c1","type":"function",
|
||||
"function":{"name":"get_weather","arguments":"{\"city\":\"Istanbul\"}"}}]}}]}`)
|
||||
}))
|
||||
admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode([]adminapi.ReplicaView{
|
||||
{ID: "r1", Model: "m", Profile: "gpu", Endpoint: engine.URL, EngineKey: "engine-key"},
|
||||
})
|
||||
}))
|
||||
t.Cleanup(func() { engine.Close(); admin.Close() })
|
||||
t.Setenv("LLMGW_ADMIN_TOKEN", "t")
|
||||
cmd := newToolCheckCmd(func() *ctlClient { return newCtlClient(admin.URL) })
|
||||
var out bytes.Buffer
|
||||
cmd.SetOut(&out)
|
||||
cmd.SetErr(&out)
|
||||
cmd.SetArgs(append([]string{"--endpoint", engine.URL}, args...))
|
||||
err := cmd.ExecuteContext(context.Background())
|
||||
return out.String(), err
|
||||
}
|
||||
|
||||
func TestToolCheckAsksWithTheEngineKeyTheAdminAPIGives(t *testing.T) {
|
||||
// Every replica's engine demands its key, and the check was sent without
|
||||
// one: it failed with a 401 on every engine in the cluster.
|
||||
out, err := toolCheck(t)
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(out, "tool calls work") {
|
||||
t.Errorf("unexpected report:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestToolCheckSaysWhereTheEngineKeyComesFrom(t *testing.T) {
|
||||
out, err := toolCheck(t, "--key", "wrong")
|
||||
if err == nil || !strings.Contains(err.Error(), "401") || !strings.Contains(err.Error(), "--key") {
|
||||
t.Fatalf("expected the refusal explained, got %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStructuredToolCallIsRecognised(t *testing.T) {
|
||||
p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
|
||||
"tool_calls":[{"id":"c1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Istanbul\"}"}}]}}]}`))
|
||||
|
||||
@@ -679,6 +679,8 @@ func (m *consoleModel) doEntry() tea.Cmd {
|
||||
}}
|
||||
case cBackup:
|
||||
return m.runAfter("server", "backup", "--config="+m.file)
|
||||
case cJudge:
|
||||
return m.runAfter("ctl", server, "guard", "judge", "enable", "--replace")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -45,6 +45,7 @@ const (
|
||||
cUsage
|
||||
cQuotas
|
||||
cGuard
|
||||
cKB
|
||||
cFederation
|
||||
cMirror
|
||||
cJobs
|
||||
@@ -59,6 +60,7 @@ const (
|
||||
cAddControl
|
||||
cReplicate
|
||||
cServe
|
||||
cJudge
|
||||
cUpgrade
|
||||
cRestartAll
|
||||
cBackup
|
||||
@@ -67,8 +69,8 @@ const (
|
||||
cCount
|
||||
)
|
||||
|
||||
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual",
|
||||
"server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Upgrade the cluster",
|
||||
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Knowledge", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual",
|
||||
"server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Serve the guardrails' judge", "Upgrade the cluster",
|
||||
"Restart all replicas", "Back up now", "Wipe this machine", "Leave"}
|
||||
|
||||
// consoleAction is what is done once the screen closes: each runs its own
|
||||
@@ -176,6 +178,7 @@ type consoleModel struct {
|
||||
quotas []adminapi.QuotaView
|
||||
quotasErr error
|
||||
guard guardState
|
||||
kb kbState
|
||||
|
||||
st *storeHolder
|
||||
people peopleData
|
||||
@@ -322,6 +325,9 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
if m.menu == cGuard {
|
||||
cmds = append(cmds, m.fetchGuard())
|
||||
}
|
||||
if m.menu == cKB {
|
||||
cmds = append(cmds, m.fetchKB())
|
||||
}
|
||||
if m.menu == cMirror {
|
||||
cmds = append(cmds, m.fetchMirror())
|
||||
}
|
||||
@@ -380,6 +386,8 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
m.people = peopleData(msg)
|
||||
case consoleQuotasMsg:
|
||||
m.quotas, m.quotasErr = msg.rows, msg.err
|
||||
case consoleKBMsg:
|
||||
m.kb = kbState{view: msg.view, err: msg.err, loaded: true}
|
||||
case consoleGuardMsg:
|
||||
window := m.guard.window
|
||||
m.guard = guardState{view: msg.view, events: msg.events, err: msg.err, window: window}
|
||||
@@ -496,7 +504,7 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
return m.leave()
|
||||
case cWipe:
|
||||
return m.askWipe()
|
||||
case cUpgrade, cRestartAll, cBackup:
|
||||
case cUpgrade, cRestartAll, cBackup, cJudge:
|
||||
return m, m.doEntry()
|
||||
case cOverview:
|
||||
return m, nil
|
||||
@@ -538,6 +546,8 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
m.menuD = m.quotaMenu(m.quotas[m.row])
|
||||
case cGuard:
|
||||
m.menuD = m.guardAttachmentMenu(m.guard.view.Attachments[m.row])
|
||||
case cKB:
|
||||
m.menuD = m.kbMenu(m.kb.view.Collections[m.row])
|
||||
case cMirror:
|
||||
m.menuD = m.mirrorMenu(m.mirrorGroups()[m.row])
|
||||
case cJobs:
|
||||
@@ -595,6 +605,10 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
m.quotaForm(adminapi.QuotaView{})
|
||||
case cGuard:
|
||||
m.guardAttachForm()
|
||||
case cKB:
|
||||
if m.kb.view.Available {
|
||||
m.kbCreateForm()
|
||||
}
|
||||
case cModels:
|
||||
m.addModelForm()
|
||||
case cMirror:
|
||||
@@ -650,6 +664,8 @@ func (m consoleModel) onEnter() tea.Cmd {
|
||||
return m.fetchQuotas()
|
||||
case cGuard:
|
||||
return m.fetchGuard()
|
||||
case cKB:
|
||||
return m.fetchKB()
|
||||
case cMirror:
|
||||
return m.fetchMirror()
|
||||
case cMonitor:
|
||||
@@ -1026,6 +1042,8 @@ func (m consoleModel) rowCount() int {
|
||||
return len(m.quotas)
|
||||
case cGuard:
|
||||
return len(m.guard.view.Attachments)
|
||||
case cKB:
|
||||
return len(m.kb.view.Collections)
|
||||
case cMirror:
|
||||
return len(m.mirrorGroups())
|
||||
case cJobs:
|
||||
@@ -1229,13 +1247,17 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
for _, s := range r.Shards {
|
||||
nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, discovery.DeviceLabels(s.GPUs)))
|
||||
}
|
||||
rows = append(rows, []string{r.ID, r.Model, r.State, r.Pool, strings.Join(nodes, " "),
|
||||
compute := "-"
|
||||
if len(r.Shards) > 0 {
|
||||
compute = r.Shards[0].Compute.Short()
|
||||
}
|
||||
rows = append(rows, []string{r.ID, r.Model, r.State, r.Pool, compute, strings.Join(nodes, " "),
|
||||
(time.Duration(r.AgeSec) * time.Second).Round(time.Second).String()})
|
||||
}
|
||||
if m.inPanel {
|
||||
keys = rowKeys
|
||||
}
|
||||
return "Replicas", "\n" + table([]string{"REPLICA", "MODEL", "STATE", "POOL", "ON", "AGE"}, rows, m.row, pw, m.inPanel), keys
|
||||
return "Replicas", "\n" + table([]string{"REPLICA", "MODEL", "STATE", "POOL", "COMPUTE", "ON", "AGE"}, rows, m.row, pw, m.inPanel), keys
|
||||
case cUsage:
|
||||
head := "\n Last 24 hours, by " + lipgloss.NewStyle().Bold(true).Render(usageGroupings[m.usageBy]) +
|
||||
sDim.Render(" (→ to group by "+usageGroupings[(m.usageBy+1)%len(usageGroupings)]+")") + "\n\n"
|
||||
@@ -1255,6 +1277,8 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
return m.quotasPanel(pw)
|
||||
case cGuard:
|
||||
return m.guardPanel(pw)
|
||||
case cKB:
|
||||
return m.kbPanel(pw)
|
||||
case cAccounts, cKeys, cOrgs:
|
||||
return m.peoplePanel(pw)
|
||||
case cManual:
|
||||
@@ -1299,6 +1323,9 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
item = "\n Moves the store onto two machines under Patroni, replicated,\n so losing one loses nothing.\n\n " +
|
||||
sDim.Render("On the command line:") +
|
||||
"\n sudo --preserve-env=SSH_AUTH_SOCK llmgw server init \\\n --patroni <user>@<a> --patroni <user>@<b> --no-mirror --no-node --no-model"
|
||||
case cJudge:
|
||||
item = "\n Serves the model the guardrail policies ask as their judge — Llama\n Guard 3 1B on CPUs, under the name the policies give it — so what they\n judge is not refused (guard_unavailable). A model of that name that is\n not a judge is replaced. Overview says when a judge is missing.\n\n " +
|
||||
sDim.Render("On the command line:") + "\n llmgw ctl guard judge status\n llmgw ctl guard judge enable --replace"
|
||||
case cUpgrade:
|
||||
item = "\n Moves the cluster to a release: this machine first, then each node\n in turn. Engines keep serving; a node that does not come back stops it.\n\n " +
|
||||
sDim.Render("On the command line:") + "\n sudo llmgw ctl upgrade VERSION"
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// Knowledge bases, as the console shows them: each collection with its
|
||||
// owner, model, documents and grants; Enter for what can be done to one, A
|
||||
// to make one. People fill them from their own machines with llmgw-kb.
|
||||
|
||||
type kbState struct {
|
||||
view adminapi.KBView
|
||||
err error
|
||||
// loaded is set once an answer came, so "none yet" is not shown while
|
||||
// asking.
|
||||
loaded bool
|
||||
}
|
||||
|
||||
type consoleKBMsg struct {
|
||||
view adminapi.KBView
|
||||
err error
|
||||
}
|
||||
|
||||
func (m consoleModel) fetchKB() tea.Cmd {
|
||||
c := m.client
|
||||
return func() tea.Msg {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
var msg consoleKBMsg
|
||||
msg.err = c.get(ctx, "/api/v1/kb", &msg.view)
|
||||
return msg
|
||||
}
|
||||
}
|
||||
|
||||
func (m consoleModel) kbPanel(pw int) (string, string, string) {
|
||||
keys := "→ or Enter open · ↑↓ choose · Q leave"
|
||||
if m.inPanel {
|
||||
keys = "↑↓ collection · Enter actions · A new collection · ← back · Q leave"
|
||||
}
|
||||
k := m.kb
|
||||
var b strings.Builder
|
||||
b.WriteString("\n " + sDim.Render("Documents requests are answered from; people add them with llmgw-kb.") + "\n\n")
|
||||
switch {
|
||||
case k.err != nil:
|
||||
b.WriteString(" " + sBad.Render(k.err.Error()))
|
||||
return "Knowledge", b.String(), keys
|
||||
case !k.loaded:
|
||||
b.WriteString(" " + sDim.Render("asking the control plane…"))
|
||||
return "Knowledge", b.String(), keys
|
||||
case !k.view.Available:
|
||||
b.WriteString(" " + sBad.Render("Off: ") + k.view.Reason)
|
||||
return "Knowledge", b.String(), keys
|
||||
case len(k.view.Collections) == 0:
|
||||
b.WriteString(" " + sDim.Render("No collections yet. A to make one."))
|
||||
return "Knowledge", b.String(), keys
|
||||
}
|
||||
var rows [][]string
|
||||
for _, c := range k.view.Collections {
|
||||
var grants []string
|
||||
for _, g := range c.Grants {
|
||||
grants = append(grants, or(g.Name, g.Target)+" "+g.Perm)
|
||||
}
|
||||
rows = append(rows, []string{or(c.OrgName, c.Org) + "/" + c.Name, or(c.OwnerName, c.Owner),
|
||||
fmt.Sprintf("%s (%d)", c.EmbedModel, c.Dims), fmt.Sprint(c.Documents), fmt.Sprint(c.Chunks), dash(strings.Join(grants, ", "))})
|
||||
}
|
||||
b.WriteString(table([]string{"COLLECTION", "OWNER", "MODEL", "DOCUMENTS", "CHUNKS", "GRANTED TO"}, rows, m.row, pw, m.inPanel))
|
||||
if len(k.view.Defaults) > 0 {
|
||||
b.WriteString("\n\n " + sTitle.Render("BY DEFAULT") + " " + sDim.Render("when a request names no collection") + "\n")
|
||||
var drows [][]string
|
||||
for _, d := range k.view.Defaults {
|
||||
drows = append(drows, []string{or(d.Name, d.Target), strings.Join(d.Collections, ", ")})
|
||||
}
|
||||
b.WriteString(table([]string{"FOR", "ANSWERED FROM"}, drows, -1, pw, false))
|
||||
}
|
||||
b.WriteString("\n\n " + sDim.Render("On a person's machine: llmgw-kb sync COLLECTION FOLDER"))
|
||||
return "Knowledge", b.String(), keys
|
||||
}
|
||||
|
||||
// kbRef is how the admin API is told which collection: by its ID.
|
||||
func kbRef(c adminapi.KBCollectionView) string { return url.PathEscape(c.ID) }
|
||||
|
||||
// kbCreateForm makes a collection.
|
||||
func (m *consoleModel) kbCreateForm() {
|
||||
model := ""
|
||||
for _, md := range m.data.models {
|
||||
n := strings.ToLower(md.Name)
|
||||
if strings.Contains(n, "embed") || strings.Contains(n, "bge") || strings.Contains(n, "e5") {
|
||||
model = md.Name
|
||||
break
|
||||
}
|
||||
}
|
||||
m.openForm(&formDialog{title: "New knowledge-base collection",
|
||||
note: "Owned by part of the tree: it and everything below it may read and add to it.\n" +
|
||||
"The model must embed (capabilities.embeddings) and be able to serve now.",
|
||||
fields: []formField{
|
||||
choiceF("owner", []string{"project", "team", "org", "user"}, "team"),
|
||||
textF("which", "", "ORG/TEAM/PROJECT, ORG/TEAM, ORG, or the ID"),
|
||||
textF("org", "", "for a person's collection only"),
|
||||
textF("name", "", "how people name it to llmgw-kb"),
|
||||
textF("model", model, "the catalog model that embeds it"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
for i, what := range map[int]string{1: "which " + v[0], 3: "a name", 4: "a model"} {
|
||||
if strings.TrimSpace(v[i]) == "" {
|
||||
return nil, fmt.Errorf("%s?", what)
|
||||
}
|
||||
}
|
||||
req := adminapi.KBCreateRequest{Owner: v[0] + ":" + v[1], Org: v[2], Name: v[3], Model: v[4]}
|
||||
return m.call("New collection "+v[3], http.MethodPost, "/api/v1/kb/collections", req, func(raw json.RawMessage) string {
|
||||
var c adminapi.KBCollectionView
|
||||
_ = json.Unmarshal(raw, &c)
|
||||
return fmt.Sprintf("Collection %s, embedded by %s in %d dimensions.\n\nPeople with access fill it with:\n llmgw-kb sync %s FOLDER",
|
||||
c.Name, c.EmbedModel, c.Dims, c.Name)
|
||||
}), nil
|
||||
}})
|
||||
}
|
||||
|
||||
// kbMenu is what can be done to one collection.
|
||||
func (m consoleModel) kbMenu(c adminapi.KBCollectionView) *menuDialog {
|
||||
name := or(c.OrgName, c.Org) + "/" + c.Name
|
||||
items := []menuItem{
|
||||
{"Documents", func(m *consoleModel) tea.Cmd { return m.kbDocuments(c) }},
|
||||
{"Grant…", func(m *consoleModel) tea.Cmd {
|
||||
m.openForm(&formDialog{title: "Grant " + name,
|
||||
note: "It and everything below it may then read, or with write add to, this collection.\nNever outside " + or(c.OrgName, c.Org) + ".",
|
||||
fields: []formField{
|
||||
choiceF("to", []string{"project", "team", "org", "user", "key"}, "project"),
|
||||
textF("which", "", "ORG/TEAM/PROJECT, ORG/TEAM, ORG, or the ID"),
|
||||
choiceF("may", []string{"read", "write"}, "read"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
if strings.TrimSpace(v[1]) == "" {
|
||||
return nil, fmt.Errorf("which %s?", v[0])
|
||||
}
|
||||
req := adminapi.KBGrantRequest{Collection: c.ID, Target: v[0] + ":" + strings.TrimSpace(v[1]), Perm: v[2]}
|
||||
return m.call("Grant "+c.Name, http.MethodPost, "/api/v1/kb/grants", req, nil), nil
|
||||
}})
|
||||
return nil
|
||||
}},
|
||||
}
|
||||
for _, g := range c.Grants {
|
||||
items = append(items, menuItem{"Revoke " + or(g.Name, g.Target) + " (" + g.Perm + ")…", func(m *consoleModel) tea.Cmd {
|
||||
m.confirmD = &confirmDialog{title: "Revoke a grant on " + name,
|
||||
text: or(g.Name, g.Target) + " may then no longer " + g.Perm + " it, unless it owns it.",
|
||||
yes: func(m *consoleModel) tea.Cmd {
|
||||
return m.call("Revoke", http.MethodDelete, "/api/v1/kb/grants",
|
||||
adminapi.KBGrantRequest{Collection: c.ID, Target: g.Target}, nil)
|
||||
}}
|
||||
return nil
|
||||
}})
|
||||
}
|
||||
items = append(items,
|
||||
menuItem{"Answer from it by default…", func(m *consoleModel) tea.Cmd {
|
||||
m.openForm(&formDialog{title: "Answer from " + name + " by default",
|
||||
note: "Chat requests from there that name no collection are answered from this one.\n" +
|
||||
"Replaces that part's default; llmgw ctl kb default sets several.",
|
||||
fields: []formField{
|
||||
choiceF("for", []string{"project", "team", "org", "key"}, "team"),
|
||||
textF("which", "", "ORG/TEAM/PROJECT, ORG/TEAM, ORG, or the ID"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
if strings.TrimSpace(v[1]) == "" {
|
||||
return nil, fmt.Errorf("which %s?", v[0])
|
||||
}
|
||||
req := adminapi.KBDefaultRequest{Target: v[0] + ":" + strings.TrimSpace(v[1]), Collections: []string{c.ID}}
|
||||
return m.call("Default "+c.Name, http.MethodPost, "/api/v1/kb/defaults", req, nil), nil
|
||||
}})
|
||||
return nil
|
||||
}},
|
||||
menuItem{"Try a search…", func(m *consoleModel) tea.Cmd {
|
||||
m.openForm(&formDialog{title: "Search " + name,
|
||||
note: "As an administrator: no tenancy applies. Shows what a question would find.",
|
||||
fields: []formField{textF("question", "", ""), textF("results", "5", "how many chunks")},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
if v[0] == "" {
|
||||
return nil, fmt.Errorf("what question?")
|
||||
}
|
||||
k, _ := strconv.Atoi(v[1])
|
||||
req := adminapi.KBSearchRequest{Collection: c.ID, Query: v[0], K: k}
|
||||
return m.call("Search "+c.Name, http.MethodPost, "/api/v1/kb/search", req, func(raw json.RawMessage) string {
|
||||
var hits []adminapi.KBHitView
|
||||
_ = json.Unmarshal(raw, &hits)
|
||||
if len(hits) == 0 {
|
||||
return "Nothing found."
|
||||
}
|
||||
var b strings.Builder
|
||||
for i, h := range hits {
|
||||
text := h.Text
|
||||
if r := []rune(text); len(r) > 400 {
|
||||
text = string(r[:400]) + "…"
|
||||
}
|
||||
fmt.Fprintf(&b, "%d. %s #%d\n%s\n\n", i+1, h.Document, h.Ord, indent(strings.TrimSpace(text), " "))
|
||||
}
|
||||
return strings.TrimRight(b.String(), "\n")
|
||||
}), nil
|
||||
}})
|
||||
return nil
|
||||
}},
|
||||
menuItem{"Delete…", func(m *consoleModel) tea.Cmd {
|
||||
m.confirmD = &confirmDialog{title: "Delete " + name,
|
||||
text: fmt.Sprintf("Its %d documents and %d chunks, and every grant, go with it. People's own files are not touched.",
|
||||
c.Documents, c.Chunks),
|
||||
yes: func(m *consoleModel) tea.Cmd {
|
||||
return m.call("Delete "+c.Name, http.MethodDelete, "/api/v1/kb/collections/"+kbRef(c), nil, nil)
|
||||
}}
|
||||
return nil
|
||||
}},
|
||||
)
|
||||
return &menuDialog{title: name, items: items}
|
||||
}
|
||||
|
||||
// kbDocuments shows a collection's documents in a dialog.
|
||||
func (m *consoleModel) kbDocuments(c adminapi.KBCollectionView) tea.Cmd {
|
||||
return m.call("Documents of "+c.Name, http.MethodGet, "/api/v1/kb/collections/"+kbRef(c)+"/documents", nil,
|
||||
func(raw json.RawMessage) string {
|
||||
var docs []adminapi.KBDocumentView
|
||||
_ = json.Unmarshal(raw, &docs)
|
||||
if len(docs) == 0 {
|
||||
return "None yet. On a person's machine: llmgw-kb sync " + c.Name + " FOLDER"
|
||||
}
|
||||
var b strings.Builder
|
||||
for i, d := range docs {
|
||||
if i == 40 {
|
||||
fmt.Fprintf(&b, "… and %d more (llmgw ctl kb docs %s)", len(docs)-40, c.ID)
|
||||
break
|
||||
}
|
||||
fmt.Fprintf(&b, "%-48s %5d chunks %s %s\n", d.Name, d.Chunks, d.AddedAt.Local().Format("2006-01-02 15:04"), d.AddedBy)
|
||||
}
|
||||
return strings.TrimRight(b.String(), "\n")
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
func TestACollectionIsMadeAndGrantedFromTheConsole(t *testing.T) {
|
||||
m, s := consoleWith(t, cKB)
|
||||
m.data.models = append(m.data.models, adminapi.ModelView{Name: "bge-m3"})
|
||||
m.kb = kbState{loaded: true, view: adminapi.KBView{Available: true, Collections: []adminapi.KBCollectionView{{
|
||||
ID: "kb_aaaaaaaaaaaaaaaa", Name: "policies", Org: "o1", OrgName: "Acme", Owner: "team:t1", OwnerName: "Acme / hr",
|
||||
EmbedModel: "bge-m3", Dims: 1024, Documents: 3, Chunks: 40,
|
||||
Grants: []adminapi.KBGrantView{{Target: "project:p9", Name: "Acme/sales/crm", Perm: "read"}},
|
||||
}}}}
|
||||
var tm tea.Model = m
|
||||
view := tm.View()
|
||||
for _, want := range []string{"Knowledge", "Acme/policies", "Acme / hr", "bge-m3 (1024)", "Acme/s", "llmgw-kb sync COLLECTION FOLDER"} {
|
||||
if !strings.Contains(view, want) {
|
||||
t.Errorf("missing %q in:\n%s", want, view)
|
||||
}
|
||||
}
|
||||
|
||||
// A: the form, prefilled with the embedding model.
|
||||
tm = press(tm, "a")
|
||||
f := tm.(consoleModel).formD
|
||||
if f == nil || f.fields[4].value() != "bge-m3" {
|
||||
t.Fatalf("no form, or the model not offered: %+v", f)
|
||||
}
|
||||
f.fields[1].text.SetValue("Acme/hr")
|
||||
f.fields[3].text.SetValue("handbook")
|
||||
tm = pressRun(tm, "tab", "tab", "tab", "tab", "tab", "enter")
|
||||
var req adminapi.KBCreateRequest
|
||||
got := s.last()
|
||||
if !strings.HasPrefix(got, "POST /api/v1/kb/collections ") ||
|
||||
json.Unmarshal([]byte(strings.SplitN(got, " ", 3)[2]), &req) != nil ||
|
||||
req.Owner != "team:Acme/hr" || req.Name != "handbook" || req.Model != "bge-m3" {
|
||||
t.Fatalf("sent %q", got)
|
||||
}
|
||||
|
||||
// Enter on a row: grant to a project for writing.
|
||||
m.formD = nil
|
||||
tm = m
|
||||
tm = press(tm, "enter")
|
||||
if md := tm.(consoleModel).menuD; md == nil || !strings.Contains(md.items[2].label, "Revoke Acme/sales/crm") {
|
||||
t.Fatalf("menu %+v", md)
|
||||
}
|
||||
tm = press(tm, "down", "enter")
|
||||
f = tm.(consoleModel).formD
|
||||
f.fields[1].text.SetValue("Acme/sales/crm")
|
||||
f.fields[2].opt = 1 // write
|
||||
tm = pressRun(tm, "tab", "tab", "tab", "enter")
|
||||
if got := s.last(); got != `POST /api/v1/kb/grants {"collection":"kb_aaaaaaaaaaaaaaaa","target":"project:Acme/sales/crm","perm":"write"}` {
|
||||
t.Fatalf("sent %q", got)
|
||||
}
|
||||
_ = tm
|
||||
}
|
||||
+53
-4
@@ -3,6 +3,7 @@ package cli
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
@@ -34,7 +35,7 @@ func newUsersCmd() *cobra.Command {
|
||||
"Stop an account working, keeping it and its history"),
|
||||
newUserStateCmd(open, "reinstate", store.UserActive,
|
||||
"Let a suspended account work again"),
|
||||
newUserCreateCmd(open), newUserResetPasswordCmd(open),
|
||||
newUserCreateCmd(open), newUserResetPasswordCmd(open), newUserResetLinkCmd(open, &sf),
|
||||
newUserJoinCmd(open), newUserLeaveCmd(open), newUserDeleteCmd(open), newUserForgetCmd(open),
|
||||
newUserAdminCmd(open, "promote", true, "Let an account see the cluster dashboard at /admin"),
|
||||
newUserAdminCmd(open, "demote", false, "Take away access to the cluster dashboard"),
|
||||
@@ -254,9 +255,10 @@ func newUserResetPasswordCmd(open userOpener) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "reset-password <user>",
|
||||
Short: "Set a new password for someone who lost theirs",
|
||||
Long: "There is no self-service reset, because this gateway sends no email and\n" +
|
||||
"a reset link with nowhere to send it is not a reset. Set one here, tell\n" +
|
||||
"them out of band, and have them change it from their profile.\n\n" +
|
||||
Long: "People reset a forgotten password themselves at /forgot when the cluster\n" +
|
||||
"has mail settings (mail.smtp); without them, 'llmgw users reset-link' makes\n" +
|
||||
"a link to hand them. Or set one here, tell them out of band, and have them\n" +
|
||||
"change it from their profile.\n\n" +
|
||||
"Every session they have open is closed: if they lost the password, the\n" +
|
||||
"reason might be that somebody else has it.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
@@ -532,3 +534,50 @@ func newUserForgetCmd(open userOpener) *cobra.Command {
|
||||
cmd.Flags().BoolVar(&yes, "yes", false, "confirm")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newUserResetLinkCmd(open userOpener, sf *storeFlags) *cobra.Command {
|
||||
var base string
|
||||
cmd := &cobra.Command{
|
||||
Use: "reset-link <user>",
|
||||
Short: "Make a one-time link for someone to set a forgotten password",
|
||||
Long: "For a gateway that sends no mail, or a person whose mail does not arrive:\n" +
|
||||
"the link is printed, to hand them however suits. It works once, for an\n" +
|
||||
"hour, and a newer one replaces it. Setting the password closes every\n" +
|
||||
"session they have open. The address it points at is mail.base_url from\n" +
|
||||
"the cluster's settings, or --base-url.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if base == "" {
|
||||
if cfg, err := loadEffectiveConfig(sf.config); err == nil {
|
||||
base = cfg.Mail.BaseURL
|
||||
}
|
||||
}
|
||||
if base == "" {
|
||||
return errors.New("give --base-url, the address people reach this gateway at, or set mail.base_url " +
|
||||
"(llmgw ctl config set mail.base_url https://…)")
|
||||
}
|
||||
s, err := open(cmd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer s.Close()
|
||||
u, err := resolveUser(cmd.Context(), s, args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if u.PasswordHash == "" {
|
||||
return fmt.Errorf("%s signs in through single sign-on: there is no password to set", u.Email)
|
||||
}
|
||||
token, err := s.CreatePasswordReset(cmd.Context(), u.ID, account.ResetLinkTTL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s/reset?token=%s\n", strings.TrimRight(base, "/"), url.QueryEscape(token))
|
||||
cmd.PrintErrf("For %s: works once, for %s. Hand it to them alone; it opens their account.\n",
|
||||
u.Email, account.ResetLinkTTL)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&base, "base-url", "", "the address people reach this gateway at (default: mail.base_url)")
|
||||
return cmd
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mail"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/secrets"
|
||||
@@ -146,6 +147,14 @@ type Server struct {
|
||||
// Prometheus and Alertmanager in between. Empty sends nothing.
|
||||
Notify notify.Config `koanf:"notify"`
|
||||
|
||||
// Mail is where the gateway's few emails go out from: a link to set a
|
||||
// forgotten password. Unset, none is sent and people ask an operator.
|
||||
Mail mail.Config `koanf:"mail"`
|
||||
|
||||
// Serving stops and starts the whole cluster's service (llmgw stop, llmgw
|
||||
// start).
|
||||
Serving ServingSettings `koanf:"serving"`
|
||||
|
||||
// Firewall turns llmgw's firewall on every machine on or off.
|
||||
Firewall FirewallSettings `koanf:"firewall"`
|
||||
|
||||
@@ -914,6 +923,9 @@ func (s Server) Validate() error {
|
||||
return fmt.Errorf("backup_copy_to needs backup_recipients: an archive holds the certificate authority's key " +
|
||||
"and every secret in the store, and leaves this machine only encrypted")
|
||||
}
|
||||
if err := s.Mail.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.FirewallAllow(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
package config
|
||||
|
||||
// ServingSettings is whether the cluster serves at all.
|
||||
type ServingSettings struct {
|
||||
// Stopped takes every model down and answers every request for one with
|
||||
// 503 (system_stopped), leaving the control planes, agents and mirror
|
||||
// running and every model's definition as it is: llmgw stop sets it,
|
||||
// llmgw start clears it, and the models come back as their catalog
|
||||
// entries ask. In force on every control plane within seconds.
|
||||
Stopped bool `koanf:"stopped"`
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
@@ -67,8 +68,12 @@ type API struct {
|
||||
// nil without a store.
|
||||
federation *federation.Service
|
||||
// guard is the guardrail policies' library and attachments.
|
||||
guard *GuardAdmin
|
||||
guard *GuardAdmin
|
||||
// kb is knowledge bases; nil without a store.
|
||||
kb *KBAdmin
|
||||
settings *Settings
|
||||
// restartAgents tells every connected agent to start again.
|
||||
restartAgents func(reason string) []string
|
||||
// revoke records a removed node, so its certificate is refused.
|
||||
revoke func(ctx context.Context, nodeID string, at time.Time) error
|
||||
|
||||
@@ -111,8 +116,10 @@ func (a *API) Register(mux Mux) {
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
|
||||
a.registerQuotas(mux)
|
||||
a.registerKB(mux)
|
||||
a.registerGuard(mux)
|
||||
a.registerSettings(mux)
|
||||
mux.HandleFunc("POST /api/v1/agents/restart", a.restartAgentsHandler)
|
||||
mux.HandleFunc("GET /api/v1/audit", a.auditEvents)
|
||||
mux.HandleFunc("GET /api/v1/usage/export", a.usageExport)
|
||||
mux.HandleFunc("POST /api/v1/notify/test", a.notifyTest)
|
||||
@@ -228,6 +235,17 @@ type NodeView struct {
|
||||
// which is not the same as having none — placement treats it that way too,
|
||||
// or a control plane upgrade would empty a fleet.
|
||||
Engines []string `json:"engines,omitempty"`
|
||||
// EngineVersions are the installed engines with their versions:
|
||||
// llamacpp → b10991. ComputeCapabilities are its GPUs' CUDA compute
|
||||
// capabilities, 7.0 for a V100, so an engine build's support can be read
|
||||
// against them.
|
||||
EngineVersions map[string]string `json:"engine_versions,omitempty"`
|
||||
// MultiNode is whether it can take part in replicas spanning nodes:
|
||||
// "ready", "down: mlx5_1" naming the declared adapters whose ports are
|
||||
// not active, or empty for a node on no fabric. Its own replicas run
|
||||
// either way.
|
||||
MultiNode string `json:"multi_node,omitempty"`
|
||||
ComputeCapabilities []string `json:"compute_capabilities,omitempty"`
|
||||
// Provision is where installing each wanted engine stands. An engine is in
|
||||
// Engines only once its installation here is ready.
|
||||
Provision []inventory.EngineInstall `json:"provision,omitempty"`
|
||||
@@ -416,10 +434,12 @@ type ModelView struct {
|
||||
Description string `json:"description,omitempty"`
|
||||
Aliases []string `json:"aliases,omitempty"`
|
||||
Source string `json:"source"`
|
||||
Min int `json:"min_replicas"`
|
||||
Max int `json:"max_replicas"`
|
||||
Ready int `json:"ready_replicas"`
|
||||
Total int `json:"total_replicas"`
|
||||
// Role is "guard" for a guardrails judge, empty for a model clients use.
|
||||
Role string `json:"role,omitempty"`
|
||||
Min int `json:"min_replicas"`
|
||||
Max int `json:"max_replicas"`
|
||||
Ready int `json:"ready_replicas"`
|
||||
Total int `json:"total_replicas"`
|
||||
// Desired is how many it should have now: Min, or for an on-demand
|
||||
// model one while it is being asked for and none otherwise.
|
||||
Desired int `json:"desired_replicas"`
|
||||
@@ -518,6 +538,30 @@ type ShardView struct {
|
||||
// node's last heartbeat; zero when not measured.
|
||||
CPUCores float64 `json:"cpu_cores,omitempty"`
|
||||
MemoryBytes int64 `json:"memory_bytes,omitempty"`
|
||||
// Compute is where the engine said it put the model as it became
|
||||
// ready: absent when it did not say.
|
||||
Compute *ComputeView `json:"compute,omitempty"`
|
||||
}
|
||||
|
||||
// ComputeView is where an engine runs its model, by its own account.
|
||||
type ComputeView struct {
|
||||
// Backend is CUDA, Vulkan, Metal, ROCm, GPU or CPU.
|
||||
Backend string `json:"backend"`
|
||||
Devices []string `json:"devices,omitempty"`
|
||||
// LayersOffloaded of LayersTotal went to the devices; absent when unsaid.
|
||||
LayersOffloaded int `json:"layers_offloaded,omitempty"`
|
||||
LayersTotal int `json:"layers_total,omitempty"`
|
||||
}
|
||||
|
||||
// Short is the compute in a few characters: "CUDA 49/49", "CPU".
|
||||
func (c *ComputeView) Short() string {
|
||||
if c == nil {
|
||||
return "-"
|
||||
}
|
||||
if c.LayersTotal > 0 && c.Backend != "CPU" {
|
||||
return fmt.Sprintf("%s %d/%d", c.Backend, c.LayersOffloaded, c.LayersTotal)
|
||||
}
|
||||
return c.Backend
|
||||
}
|
||||
|
||||
// FetchView is a download in flight.
|
||||
@@ -623,6 +667,16 @@ func (a *API) Nodes() []NodeView {
|
||||
v.Engines = append(v.Engines, name)
|
||||
}
|
||||
slices.Sort(v.Engines)
|
||||
if len(n.EngineVersions) > 0 {
|
||||
v.EngineVersions = maps.Clone(n.EngineVersions)
|
||||
}
|
||||
for _, g := range n.GPUs {
|
||||
if g.ComputeCapability != "" && !slices.Contains(v.ComputeCapabilities, g.ComputeCapability) {
|
||||
v.ComputeCapabilities = append(v.ComputeCapabilities, g.ComputeCapability)
|
||||
}
|
||||
}
|
||||
slices.Sort(v.ComputeCapabilities)
|
||||
v.MultiNode = multiNodeReadiness(n)
|
||||
v.Provision = n.Provision
|
||||
if ch := a.ctl.EngineChoices(); ch != nil && len(n.GPUs) > 0 {
|
||||
chosen := ch.For(n.ID)
|
||||
@@ -761,6 +815,7 @@ func (a *API) Models() []ModelView {
|
||||
Class: m.EffectiveClass(), Owner: m.Owner, Preempted: h.Preempted, HeldFor: h.HeldFor,
|
||||
External: externalLabel(m), Adapters: m.Adapters,
|
||||
Canary: m.Canary, CanaryOf: a.ctl.Catalog().CanaryOf(m.Name), Eval: m.Eval,
|
||||
Role: m.Role,
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
v.Profiles = append(v.Profiles, p.Name)
|
||||
@@ -829,6 +884,10 @@ func (a *API) Replicas(model string) []ReplicaView {
|
||||
sv.Fetch.Percent = &p
|
||||
}
|
||||
}
|
||||
if c := s.Compute; c != nil {
|
||||
sv.Compute = &ComputeView{Backend: c.Backend, Devices: c.Devices,
|
||||
LayersOffloaded: c.Offloaded, LayersTotal: c.Total}
|
||||
}
|
||||
if l := loads[s.NodeID]; l != nil {
|
||||
if use, ok := l.Shards[s.ID]; ok {
|
||||
sv.CPUCores, sv.MemoryBytes = use.CPUCores, use.RSSBytes
|
||||
@@ -1473,6 +1532,11 @@ type RestartRequest struct {
|
||||
// InPlace stops each replica before starting its replacement, rather
|
||||
// than starting the replacement beside it first.
|
||||
InPlace bool `json:"in_place,omitempty"`
|
||||
// MaxSurge is how many of a model's replicas are replaced beside
|
||||
// themselves at once, MaxUnavailable how many are stopped first at once;
|
||||
// both zero is one at a time, replaced beside itself.
|
||||
MaxSurge int `json:"max_surge,omitempty"`
|
||||
MaxUnavailable int `json:"max_unavailable,omitempty"`
|
||||
}
|
||||
|
||||
// RestartResult lists the replicas queued.
|
||||
@@ -1496,11 +1560,12 @@ func (a *API) restart(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusBadRequest, "nothing ready matches; name replicas, models, or all")
|
||||
return
|
||||
}
|
||||
restart := a.ctl.Reconciler().Restart
|
||||
if req.InPlace {
|
||||
restart = a.ctl.Reconciler().RestartInPlace
|
||||
if req.MaxSurge < 0 || req.MaxUnavailable < 0 {
|
||||
writeError(w, http.StatusBadRequest, "max_surge and max_unavailable are counts of replicas, not negative")
|
||||
return
|
||||
}
|
||||
if err := restart(ids...); err != nil {
|
||||
pace := reconciler.RestartPace{MaxSurge: req.MaxSurge, MaxUnavailable: req.MaxUnavailable}
|
||||
if err := a.ctl.Reconciler().RestartPaced(pace, req.InPlace, ids...); err != nil {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
@@ -1545,6 +1610,7 @@ type RequestView struct {
|
||||
GuardAction string `json:"guard_action,omitempty"`
|
||||
GuardPolicy string `json:"guard_policy,omitempty"`
|
||||
GuardCategory string `json:"guard_category,omitempty"`
|
||||
GuardRule string `json:"guard_rule,omitempty"`
|
||||
GuardMs int64 `json:"guard_ms,omitempty"`
|
||||
// What the client said about it: OpenAI's user field, and its tags.
|
||||
EndUser string `json:"end_user,omitempty"`
|
||||
@@ -1573,7 +1639,7 @@ func (a *API) request(w http.ResponseWriter, r *http.Request) {
|
||||
Model: u.Model, ServedModel: u.ServedModel, Replica: u.Replica, Status: u.Status,
|
||||
LatencyMs: u.LatencyMs, QueueMs: u.QueueMs, TTFTMs: u.TTFTMs,
|
||||
PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros,
|
||||
GuardAction: u.GuardAction, GuardPolicy: u.GuardPolicy, GuardCategory: u.GuardCategory, GuardMs: u.GuardMs,
|
||||
GuardAction: u.GuardAction, GuardPolicy: u.GuardPolicy, GuardCategory: u.GuardCategory, GuardRule: u.GuardRule, GuardMs: u.GuardMs,
|
||||
EndUser: u.EndUser, Tags: u.Tags, CacheHit: u.CacheHit,
|
||||
})
|
||||
}
|
||||
@@ -1975,3 +2041,32 @@ func (a *API) SetNodeEngine(ctx context.Context, node, engine, build, by string)
|
||||
}
|
||||
return ch.Set(ctx, node, engine, build, by)
|
||||
}
|
||||
|
||||
// multiNodeReadiness is whether a node's declared fabrics are up: every
|
||||
// adapter they name with an active port. Empty for a node on no fabric.
|
||||
func multiNodeReadiness(n inventory.Node) string {
|
||||
if len(n.Fabrics) == 0 {
|
||||
return ""
|
||||
}
|
||||
if len(n.IBPorts) == 0 {
|
||||
return "unknown: no port reported yet"
|
||||
}
|
||||
var down []string
|
||||
for _, f := range n.Fabrics {
|
||||
for _, hca := range f.HCAs {
|
||||
up := false
|
||||
for _, p := range n.IBPorts {
|
||||
if p.HCA == hca && p.Active() {
|
||||
up = true
|
||||
}
|
||||
}
|
||||
if !up && !slices.Contains(down, hca) {
|
||||
down = append(down, hca)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(down) > 0 {
|
||||
return "down: " + strings.Join(down, ", ")
|
||||
}
|
||||
return "ready"
|
||||
}
|
||||
@@ -28,6 +28,10 @@ type GuardAdmin struct {
|
||||
Library func() guard.Config
|
||||
Store GuardStore
|
||||
Invalidate func()
|
||||
// JudgeErrors are each judge's last failure to give a verdict.
|
||||
JudgeErrors func() map[string]guard.JudgeError
|
||||
// Ask has a judge read a text, past the cache.
|
||||
Ask func(ctx context.Context, model, text, answer string) (guard.JudgeAnswer, error)
|
||||
}
|
||||
|
||||
// SetGuard enables the guard endpoints.
|
||||
@@ -35,6 +39,8 @@ func (a *API) SetGuard(g *GuardAdmin) { a.guard = g }
|
||||
|
||||
// GuardView is the library and where its policies are attached.
|
||||
type GuardView struct {
|
||||
// Enabled is whether the guardrails check anything (guard.enabled).
|
||||
Enabled bool `json:"enabled"`
|
||||
Policies []string `json:"policies"`
|
||||
Default []string `json:"default"`
|
||||
Attachments []GuardAttachmentView `json:"attachments"`
|
||||
@@ -89,7 +95,10 @@ type GuardEventView struct {
|
||||
Action string `json:"action"`
|
||||
Policy string `json:"policy"`
|
||||
Category string `json:"category,omitempty"`
|
||||
GuardMs int64 `json:"guard_ms"`
|
||||
// Rule is what decided it within the category: a pattern's name, or a
|
||||
// judge's verdict, "S6,Specialized Advice".
|
||||
Rule string `json:"rule,omitempty"`
|
||||
GuardMs int64 `json:"guard_ms"`
|
||||
}
|
||||
|
||||
// policyView sums one policy up.
|
||||
@@ -198,6 +207,8 @@ type GuardAttachmentRequest struct {
|
||||
}
|
||||
|
||||
func (a *API) registerGuard(mux Mux) {
|
||||
mux.HandleFunc("GET /api/v1/guard/judges", a.listJudges)
|
||||
mux.HandleFunc("POST /api/v1/guard/judges/ask", a.askJudge)
|
||||
mux.HandleFunc("GET /api/v1/guard", func(w http.ResponseWriter, r *http.Request) {
|
||||
if a.guard == nil {
|
||||
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)")
|
||||
@@ -208,7 +219,7 @@ func (a *API) registerGuard(mux Mux) {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
v := GuardView{Default: a.guard.Library().Default, Attachments: []GuardAttachmentView{}}
|
||||
v := GuardView{Enabled: a.guard.Library().On(), Default: a.guard.Library().Default, Attachments: []GuardAttachmentView{}}
|
||||
var names map[string]string
|
||||
if a.quotas != nil {
|
||||
names, _ = a.scopeNames(r.Context())
|
||||
@@ -269,7 +280,7 @@ func (a *API) registerGuard(mux Mux) {
|
||||
slices.SortStableFunc(v.Counts, func(a, b GuardCountView) int { return int(b.Count - a.Count) })
|
||||
for _, e := range events {
|
||||
v.Events = append(v.Events, GuardEventView{At: e.At, RequestID: e.RequestID, Project: project(e.ProjectID),
|
||||
Model: e.Model, Status: e.Status, Action: e.Action, Policy: e.Policy, Category: e.Category, GuardMs: e.GuardMs})
|
||||
Model: e.Model, Status: e.Status, Action: e.Action, Policy: e.Policy, Category: e.Category, Rule: e.Rule, GuardMs: e.GuardMs})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
})
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
// JudgeView is one judge model the guardrail policies ask, and whether it
|
||||
// can answer them.
|
||||
type JudgeView struct {
|
||||
Model string `json:"model"`
|
||||
// Policies are those that ask it; Closed those that refuse what it
|
||||
// cannot read (fail: closed).
|
||||
Policies []string `json:"policies"`
|
||||
Closed []string `json:"closed,omitempty"`
|
||||
// Then are the judges its policies ask after it, when it gives no
|
||||
// verdict: a CPU judge behind a GPU one.
|
||||
Then []string `json:"then,omitempty"`
|
||||
// InCatalog is whether a model of that name is in the catalog, and
|
||||
// Judge whether it is there as a judge (role: guard).
|
||||
InCatalog bool `json:"in_catalog"`
|
||||
Judge bool `json:"judge"`
|
||||
Ready int `json:"ready_replicas"`
|
||||
Wanted int `json:"wanted_replicas"`
|
||||
// LastError is its last failure to give a verdict, if any.
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
LastErrorAt *time.Time `json:"last_error_at,omitempty"`
|
||||
// Problem says, in words, what is wrong and what to do; empty when
|
||||
// nothing is.
|
||||
Problem string `json:"problem,omitempty"`
|
||||
}
|
||||
|
||||
// judgeErrorFresh is how recent a judge's failure must be to count as a
|
||||
// problem now.
|
||||
const judgeErrorFresh = 5 * time.Minute
|
||||
|
||||
// Judges are the judge models the policies in force ask, each with what is
|
||||
// wrong with it, if anything.
|
||||
func (a *API) Judges() []JudgeView {
|
||||
if a.guard == nil || a.guard.Library == nil {
|
||||
return nil
|
||||
}
|
||||
lib := a.guard.Library()
|
||||
if !lib.On() {
|
||||
// Off: no judge is asked, so none is missing.
|
||||
return nil
|
||||
}
|
||||
byModel := map[string]*JudgeView{}
|
||||
for name, p := range lib.Policies {
|
||||
judges := p.Judge.Judges()
|
||||
for i, model := range judges {
|
||||
v := byModel[model]
|
||||
if v == nil {
|
||||
v = &JudgeView{Model: model}
|
||||
byModel[model] = v
|
||||
}
|
||||
if !slices.Contains(v.Policies, name) {
|
||||
v.Policies = append(v.Policies, name)
|
||||
}
|
||||
if p.Fail == "closed" && p.Mode != "shadow" && !slices.Contains(v.Closed, name) {
|
||||
v.Closed = append(v.Closed, name)
|
||||
}
|
||||
for _, after := range judges[i+1:] {
|
||||
if !slices.Contains(v.Then, after) {
|
||||
v.Then = append(v.Then, after)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ready := map[string]int{}
|
||||
for _, rep := range a.ctl.Reconciler().Replicas() {
|
||||
if rep.State == reconciler.Ready {
|
||||
ready[rep.Model]++
|
||||
}
|
||||
}
|
||||
var errs map[string]guard.JudgeError
|
||||
if a.guard.JudgeErrors != nil {
|
||||
errs = a.guard.JudgeErrors()
|
||||
}
|
||||
out := make([]JudgeView, 0, len(byModel))
|
||||
for _, v := range byModel {
|
||||
sort.Strings(v.Policies)
|
||||
sort.Strings(v.Closed)
|
||||
if m, ok := a.ctl.Catalog().Resolve(v.Model); ok && m.Name == v.Model {
|
||||
v.InCatalog, v.Judge = true, m.IsGuard()
|
||||
v.Wanted, v.Ready = m.Scaling.Min, ready[m.Name]
|
||||
}
|
||||
if e, ok := errs[v.Model]; ok {
|
||||
at := e.At
|
||||
v.LastError, v.LastErrorAt = e.Err, &at
|
||||
}
|
||||
v.Problem = judgeProblem(*v)
|
||||
out = append(out, *v)
|
||||
}
|
||||
slices.SortFunc(out, func(x, y JudgeView) int {
|
||||
switch {
|
||||
case x.Model < y.Model:
|
||||
return -1
|
||||
case x.Model > y.Model:
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func judgeProblem(v JudgeView) string {
|
||||
consequence := "requests its policies judge are served unjudged"
|
||||
switch {
|
||||
case len(v.Then) > 0:
|
||||
consequence = "its policies ask " + strings.Join(v.Then, ", else ") + " instead"
|
||||
case len(v.Closed) > 0:
|
||||
consequence = "requests its policies judge are refused (guard_unavailable)"
|
||||
}
|
||||
switch {
|
||||
case !v.InCatalog:
|
||||
return fmt.Sprintf("not in the catalog, so %s: llmgw ctl guard judge enable", consequence)
|
||||
case !v.Judge:
|
||||
return fmt.Sprintf("in the catalog as a model clients use, not as a judge (role: guard), so %s: "+
|
||||
"llmgw ctl guard judge enable --replace", consequence)
|
||||
case v.Ready == 0:
|
||||
return fmt.Sprintf("no replica ready, so %s: llmgw ctl replicas --model %s", consequence, v.Model)
|
||||
case v.LastErrorAt != nil && time.Since(*v.LastErrorAt) < judgeErrorFresh:
|
||||
return fmt.Sprintf("gave no verdict %s ago (%s)", time.Since(*v.LastErrorAt).Round(time.Second), v.LastError)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (a *API) listJudges(w http.ResponseWriter, _ *http.Request) {
|
||||
if a.guard == nil {
|
||||
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, a.Judges())
|
||||
}
|
||||
|
||||
// JudgeAskRequest is a text to have a judge read.
|
||||
type JudgeAskRequest struct {
|
||||
// Model is the judge; empty is the one the policies ask, when they ask one.
|
||||
Model string `json:"model,omitempty"`
|
||||
Text string `json:"text"`
|
||||
// Answer, when given, is read as the assistant's reply to Text.
|
||||
Answer string `json:"answer,omitempty"`
|
||||
}
|
||||
|
||||
// JudgeAskView is what the judge answered.
|
||||
type JudgeAskView struct {
|
||||
Model string `json:"model"`
|
||||
// Verdict is safe, unsafe or controversial, as the gateway reads the
|
||||
// answer; a policy's judge.controversial says what controversial counts as.
|
||||
Verdict string `json:"verdict"`
|
||||
Categories []string `json:"categories,omitempty"`
|
||||
// Raw is the judge's answer as it gave it.
|
||||
Raw string `json:"raw"`
|
||||
TookMs int64 `json:"took_ms"`
|
||||
}
|
||||
|
||||
func (a *API) askJudge(w http.ResponseWriter, r *http.Request) {
|
||||
if a.guard == nil || a.guard.Ask == nil {
|
||||
writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)")
|
||||
return
|
||||
}
|
||||
var req JudgeAskRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || strings.TrimSpace(req.Text) == "" {
|
||||
writeError(w, http.StatusBadRequest, "give the text for the judge to read")
|
||||
return
|
||||
}
|
||||
if req.Model == "" {
|
||||
judges := a.Judges()
|
||||
if len(judges) == 0 {
|
||||
writeError(w, http.StatusBadRequest, "no policy asks a judge: give the model")
|
||||
return
|
||||
}
|
||||
req.Model = judges[0].Model
|
||||
}
|
||||
ans, err := a.guard.Ask(r.Context(), req.Model, req.Text, req.Answer)
|
||||
v := JudgeAskView{Model: req.Model, Raw: ans.Raw, Categories: ans.Categories, TookMs: ans.Took.Milliseconds()}
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadGateway, fmt.Sprintf("the judge %s gave no verdict: %v (it answered %q)", req.Model, err, ans.Raw))
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case ans.Unsafe:
|
||||
v.Verdict = "unsafe"
|
||||
case ans.Controversial:
|
||||
// Qwen3Guard's middle verdict: what it counts as is the policy's.
|
||||
v.Verdict = "controversial"
|
||||
default:
|
||||
v.Verdict = "safe"
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
func TestEachJudgeSaysWhatIsWrongWithIt(t *testing.T) {
|
||||
models := append(testModels(),
|
||||
// Served from the mirror page before it knew better: a model
|
||||
// clients use, under the judge's name.
|
||||
catalog.Model{Name: "shield", Source: "/s", Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}},
|
||||
catalog.Model{Name: "llama-guard-3-1b", Source: "/g", Role: catalog.RoleGuard,
|
||||
Guard: &catalog.GuardModel{Format: "llamaguard3"}, Scaling: catalog.Scaling{Min: 2, Max: 2},
|
||||
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}})
|
||||
a, _ := setup(t, models)
|
||||
lib := guard.Config{Default: []string{"baseline"}, Policies: map[string]guard.Policy{
|
||||
"baseline": {Fail: "closed", Judge: &guard.JudgePolicy{Model: "llama-guard-3-1b", Input: "block"}},
|
||||
"strict": {Fail: "closed", Judge: &guard.JudgePolicy{Model: "missing-judge", Input: "block"}},
|
||||
"shield": {Fail: "open", Judge: &guard.JudgePolicy{Model: "shield", Input: "flag"}},
|
||||
}}
|
||||
a.SetGuard(&GuardAdmin{Library: func() guard.Config { return lib }})
|
||||
|
||||
var js []JudgeView
|
||||
if code := call(t, a, "GET", "/api/v1/guard/judges", "", &js); code != 200 || len(js) != 3 {
|
||||
t.Fatalf("%d %+v", code, js)
|
||||
}
|
||||
by := map[string]JudgeView{}
|
||||
for _, j := range js {
|
||||
by[j.Model] = j
|
||||
}
|
||||
if j := by["missing-judge"]; j.InCatalog || !strings.Contains(j.Problem, "not in the catalog") ||
|
||||
!strings.Contains(j.Problem, "refused") {
|
||||
t.Errorf("missing: %+v", j)
|
||||
}
|
||||
if j := by["shield"]; !j.InCatalog || j.Judge || !strings.Contains(j.Problem, "--replace") ||
|
||||
!strings.Contains(j.Problem, "unjudged") {
|
||||
t.Errorf("not a judge: %+v", j)
|
||||
}
|
||||
if j := by["llama-guard-3-1b"]; !j.Judge || j.Wanted != 2 || !strings.Contains(j.Problem, "no replica ready") {
|
||||
t.Errorf("not running: %+v", j)
|
||||
}
|
||||
|
||||
var s Summary
|
||||
call(t, a, "GET", "/api/v1/summary", "", &s)
|
||||
if len(s.Judges) != 3 {
|
||||
t.Errorf("the summary does not carry the judges' problems: %+v", s.Judges)
|
||||
}
|
||||
|
||||
// A judge that failed lately says so, once it is otherwise fine.
|
||||
at := time.Now()
|
||||
v := JudgeView{Model: "x", InCatalog: true, Judge: true, Ready: 1, LastError: "deadline exceeded", LastErrorAt: &at}
|
||||
if p := judgeProblem(v); !strings.Contains(p, "deadline exceeded") {
|
||||
t.Errorf("a recent failure: %q", p)
|
||||
}
|
||||
old := at.Add(-time.Hour)
|
||||
v.LastErrorAt = &old
|
||||
if p := judgeProblem(v); p != "" {
|
||||
t.Errorf("an old failure still counts: %q", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAJudgeIsAskedAndItsWordsShown(t *testing.T) {
|
||||
a, _ := setup(t, testModels())
|
||||
lib := guard.Config{Policies: map[string]guard.Policy{
|
||||
"baseline": {Judge: &guard.JudgePolicy{Model: "llama-guard-3-1b", Input: "block"}}}}
|
||||
var asked string
|
||||
a.SetGuard(&GuardAdmin{Library: func() guard.Config { return lib },
|
||||
Ask: func(_ context.Context, model, text, _ string) (guard.JudgeAnswer, error) {
|
||||
asked = model + "|" + text
|
||||
return guard.JudgeAnswer{Model: model, Raw: "unsafe\nS6", Unsafe: true,
|
||||
Categories: []string{"S6", "Specialized Advice"}}, nil
|
||||
}})
|
||||
var v JudgeAskView
|
||||
if code := call(t, a, "POST", "/api/v1/guard/judges/ask", `{"text":"how do I sort a list"}`, &v); code != 200 {
|
||||
t.Fatalf("%d", code)
|
||||
}
|
||||
if asked != "llama-guard-3-1b|how do I sort a list" || v.Verdict != "unsafe" || v.Raw != "unsafe\nS6" ||
|
||||
len(v.Categories) != 2 {
|
||||
t.Errorf("asked %q, got %+v", asked, v)
|
||||
}
|
||||
if code := call(t, a, "POST", "/api/v1/guard/judges/ask", `{"text":" "}`, nil); code != 400 {
|
||||
t.Errorf("no text: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeSaysItsEnginesVersionsAndCapabilities(t *testing.T) {
|
||||
n := gpuNode("volta", 2)
|
||||
n.EngineVersions = map[string]string{"llamacpp": "b10991", "vllm": "0.10.2"}
|
||||
for i := range n.GPUs {
|
||||
n.GPUs[i].ComputeCapability = "7.0"
|
||||
}
|
||||
a, _ := setup(t, testModels(), n)
|
||||
var nodes []NodeView
|
||||
call(t, a, "GET", "/api/v1/nodes", "", &nodes)
|
||||
if len(nodes) != 1 || nodes[0].EngineVersions["vllm"] != "0.10.2" ||
|
||||
len(nodes[0].ComputeCapabilities) != 1 || nodes[0].ComputeCapabilities[0] != "7.0" {
|
||||
t.Errorf("%+v", nodes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeSaysWhetherItsFabricsAreUp(t *testing.T) {
|
||||
n := gpuNode("ib1", 2, "ib-a")
|
||||
if got := multiNodeReadiness(n); !strings.HasPrefix(got, "unknown") {
|
||||
t.Errorf("no port yet: %q", got)
|
||||
}
|
||||
n.IBPorts = []inventory.IBPort{{HCA: "mlx5_0", State: "DOWN"}}
|
||||
if got := multiNodeReadiness(n); got != "down: mlx5_0" {
|
||||
t.Errorf("down: %q", got)
|
||||
}
|
||||
n.IBPorts[0].State = "ACTIVE"
|
||||
if got := multiNodeReadiness(n); got != "ready" {
|
||||
t.Errorf("up: %q", got)
|
||||
}
|
||||
if got := multiNodeReadiness(gpuNode("plain", 1)); got != "" {
|
||||
t.Errorf("no fabric: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,545 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// Knowledge bases, administered (BACKLOG F): collections made, granted,
|
||||
// emptied and removed here; documents put into them by people with
|
||||
// llmgw-kb, through the gateway.
|
||||
|
||||
// KBStore is the part of the store knowledge bases need.
|
||||
type KBStore interface {
|
||||
KnowledgeAvailable() error
|
||||
Collections(ctx context.Context) ([]store.KBCollection, error)
|
||||
Collection(ctx context.Context, id string) (store.KBCollection, error)
|
||||
CreateCollection(ctx context.Context, c store.KBCollection) (store.KBCollection, error)
|
||||
DeleteCollection(ctx context.Context, id string) error
|
||||
Grant(ctx context.Context, g store.KBGrant) error
|
||||
Revoke(ctx context.Context, scope store.Scope, scopeID, collection string) (bool, error)
|
||||
Grants(ctx context.Context, collection string) ([]store.KBGrant, error)
|
||||
Documents(ctx context.Context, collection string) ([]store.KBDocument, error)
|
||||
DeleteDocument(ctx context.Context, collection, name string) (bool, error)
|
||||
SearchChunks(ctx context.Context, q store.KBSearch) ([]store.KBHit, error)
|
||||
SetKBDefault(ctx context.Context, d store.KBDefault) error
|
||||
KBDefaults(ctx context.Context) ([]store.KBDefault, error)
|
||||
}
|
||||
|
||||
// KBAdmin is what the knowledge-base endpoints need: the store, and a way to
|
||||
// have a model embed a text (to learn its dimension, and to try a search).
|
||||
type KBAdmin struct {
|
||||
Store KBStore
|
||||
Embed func(ctx context.Context, model, text string) ([]float32, error)
|
||||
}
|
||||
|
||||
// SetKB enables the knowledge-base endpoints.
|
||||
func (a *API) SetKB(k *KBAdmin) { a.kb = k }
|
||||
|
||||
// KBView is every collection, or why there can be none.
|
||||
type KBView struct {
|
||||
Available bool `json:"available"`
|
||||
// Reason is why not, when not.
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Collections []KBCollectionView `json:"collections"`
|
||||
// Defaults are the collections parts of the tree answer from when a
|
||||
// request names none.
|
||||
Defaults []KBDefaultView `json:"defaults"`
|
||||
}
|
||||
|
||||
// KBDefaultView is one part of the tree's default collections.
|
||||
type KBDefaultView struct {
|
||||
Target string `json:"target"`
|
||||
Name string `json:"name,omitempty"`
|
||||
// Collections are ORG/NAME, in order.
|
||||
Collections []string `json:"collections"`
|
||||
At time.Time `json:"at"`
|
||||
By string `json:"by,omitempty"`
|
||||
}
|
||||
|
||||
// KBDefaultRequest sets the collections a part of the tree answers from
|
||||
// when a request names none; none clears it.
|
||||
type KBDefaultRequest struct {
|
||||
// Target is org:, team:, project: (by path or ID) or key:KEY-ID.
|
||||
Target string `json:"target"`
|
||||
// Collections are IDs, names when unique, or ORG/NAME.
|
||||
Collections []string `json:"collections"`
|
||||
}
|
||||
|
||||
// KBCollectionView is a collection as an administrator sees it.
|
||||
type KBCollectionView struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Org string `json:"org"`
|
||||
OrgName string `json:"org_name,omitempty"`
|
||||
Owner string `json:"owner"`
|
||||
OwnerName string `json:"owner_name,omitempty"`
|
||||
EmbedModel string `json:"embed_model"`
|
||||
Dims int `json:"dims"`
|
||||
Documents int `json:"documents"`
|
||||
Chunks int `json:"chunks"`
|
||||
// Grants are who else may read or write it.
|
||||
Grants []KBGrantView `json:"grants"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
CreatedBy string `json:"created_by,omitempty"`
|
||||
}
|
||||
|
||||
// KBGrantView is one grant.
|
||||
type KBGrantView struct {
|
||||
Target string `json:"target"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Perm string `json:"perm"`
|
||||
At time.Time `json:"at"`
|
||||
By string `json:"by,omitempty"`
|
||||
}
|
||||
|
||||
// KBCreateRequest makes a collection.
|
||||
type KBCreateRequest struct {
|
||||
// Owner is org:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT (or the ID
|
||||
// after the colon), or user:USER-ID.
|
||||
Owner string `json:"owner"`
|
||||
// Org is the organization a person's collection belongs to; only for
|
||||
// user: owners.
|
||||
Org string `json:"org,omitempty"`
|
||||
Name string `json:"name"`
|
||||
// Model is the catalog model that embeds its documents.
|
||||
Model string `json:"model"`
|
||||
// Dims is the model's embedding dimension; asked of the model when 0.
|
||||
Dims int `json:"dims,omitempty"`
|
||||
}
|
||||
|
||||
// KBGrantRequest lets a scope read or write a collection, or with DELETE
|
||||
// takes that away.
|
||||
type KBGrantRequest struct {
|
||||
// Collection is its ID, its name when that is unique, or ORG/NAME.
|
||||
Collection string `json:"collection"`
|
||||
// Target is org:, team:, project:, key: or user: as for guard policies.
|
||||
Target string `json:"target"`
|
||||
// Perm is read or write; ignored by DELETE.
|
||||
Perm string `json:"perm,omitempty"`
|
||||
}
|
||||
|
||||
// KBDocumentView is one document in a collection.
|
||||
type KBDocumentView struct {
|
||||
Name string `json:"name"`
|
||||
Digest string `json:"digest"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Chunks int `json:"chunks"`
|
||||
AddedAt time.Time `json:"added_at"`
|
||||
AddedBy string `json:"added_by,omitempty"`
|
||||
}
|
||||
|
||||
// KBSearchRequest tries a search, as an administrator: no tenancy applies.
|
||||
type KBSearchRequest struct {
|
||||
Collection string `json:"collection"`
|
||||
Query string `json:"query"`
|
||||
K int `json:"k,omitempty"`
|
||||
}
|
||||
|
||||
// KBHitView is one chunk a search found.
|
||||
type KBHitView struct {
|
||||
Document string `json:"document"`
|
||||
Ord int `json:"ord"`
|
||||
Score float64 `json:"score"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
func (a *API) registerKB(mux Mux) {
|
||||
mux.HandleFunc("GET /api/v1/kb", a.kbList)
|
||||
mux.HandleFunc("POST /api/v1/kb/collections", a.kbCreate)
|
||||
mux.HandleFunc("DELETE /api/v1/kb/collections/{id}", a.kbDelete)
|
||||
mux.HandleFunc("GET /api/v1/kb/collections/{id}/documents", a.kbDocuments)
|
||||
mux.HandleFunc("DELETE /api/v1/kb/collections/{id}/documents", a.kbDeleteDocument)
|
||||
mux.HandleFunc("POST /api/v1/kb/grants", a.kbGrant(true))
|
||||
mux.HandleFunc("DELETE /api/v1/kb/grants", a.kbGrant(false))
|
||||
mux.HandleFunc("POST /api/v1/kb/search", a.kbSearch)
|
||||
mux.HandleFunc("POST /api/v1/kb/defaults", a.kbSetDefault)
|
||||
}
|
||||
|
||||
// kbReady answers for a gateway without knowledge bases, and says whether
|
||||
// it did.
|
||||
func (a *API) kbReady(w http.ResponseWriter) bool {
|
||||
if a.kb == nil {
|
||||
writeError(w, http.StatusNotImplemented, "knowledge bases need the store, which this control plane runs without")
|
||||
return false
|
||||
}
|
||||
if err := a.kb.Store.KnowledgeAvailable(); err != nil {
|
||||
writeError(w, http.StatusNotImplemented, err.Error())
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (a *API) names(ctx context.Context) map[string]string {
|
||||
if a.quotas == nil {
|
||||
return map[string]string{}
|
||||
}
|
||||
n, err := a.scopeNames(ctx)
|
||||
if err != nil {
|
||||
return map[string]string{}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (a *API) kbList(w http.ResponseWriter, r *http.Request) {
|
||||
if a.kb == nil {
|
||||
writeJSON(w, http.StatusOK, KBView{Reason: "knowledge bases need the store", Collections: []KBCollectionView{}, Defaults: []KBDefaultView{}})
|
||||
return
|
||||
}
|
||||
if err := a.kb.Store.KnowledgeAvailable(); err != nil {
|
||||
writeJSON(w, http.StatusOK, KBView{Reason: err.Error(), Collections: []KBCollectionView{}, Defaults: []KBDefaultView{}})
|
||||
return
|
||||
}
|
||||
cs, err := a.kb.Store.Collections(r.Context())
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
grants, err := a.kb.Store.Grants(r.Context(), "")
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
names := a.names(r.Context())
|
||||
v := KBView{Available: true, Collections: []KBCollectionView{}, Defaults: []KBDefaultView{}}
|
||||
for _, c := range cs {
|
||||
cv := KBCollectionView{ID: c.ID, Name: c.Name, Org: c.OrgID, OrgName: names["org:"+c.OrgID],
|
||||
Owner: string(c.OwnerScope) + ":" + c.OwnerID, OwnerName: names[string(c.OwnerScope)+":"+c.OwnerID],
|
||||
EmbedModel: c.EmbedModel, Dims: c.Dims, Grants: []KBGrantView{}, CreatedAt: c.CreatedAt, CreatedBy: c.CreatedBy}
|
||||
if docs, err := a.kb.Store.Documents(r.Context(), c.ID); err == nil {
|
||||
cv.Documents = len(docs)
|
||||
for _, d := range docs {
|
||||
cv.Chunks += d.Chunks
|
||||
}
|
||||
}
|
||||
for _, g := range grants {
|
||||
if g.Collection == c.ID {
|
||||
key := string(g.Scope) + ":" + g.ScopeID
|
||||
cv.Grants = append(cv.Grants, KBGrantView{Target: key, Name: names[key], Perm: string(g.Perm), At: g.AddedAt, By: g.AddedBy})
|
||||
}
|
||||
}
|
||||
v.Collections = append(v.Collections, cv)
|
||||
}
|
||||
label := map[string]string{}
|
||||
for _, c := range cs {
|
||||
label[c.ID] = or(names["org:"+c.OrgID], c.OrgID) + "/" + c.Name
|
||||
}
|
||||
defaults, err := a.kb.Store.KBDefaults(r.Context())
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
for _, d := range defaults {
|
||||
key := string(d.Scope) + ":" + d.ScopeID
|
||||
dv := KBDefaultView{Target: key, Name: names[key], Collections: []string{}, At: d.AddedAt, By: d.AddedBy}
|
||||
for _, id := range d.Collections {
|
||||
dv.Collections = append(dv.Collections, or(label[id], id))
|
||||
}
|
||||
v.Defaults = append(v.Defaults, dv)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
// kbTarget reads a grant's or an owner's target: org:, team: and project:
|
||||
// by path or ID, key: and user: by ID.
|
||||
func (a *API) kbTarget(ctx context.Context, target string) (store.Scope, string, error) {
|
||||
kind, what, ok := strings.Cut(strings.TrimSpace(target), ":")
|
||||
what = strings.TrimSpace(what)
|
||||
if !ok || what == "" {
|
||||
return "", "", fmt.Errorf("%q is not org:, team:, project:, key: or user: and what", target)
|
||||
}
|
||||
switch store.Scope(kind) {
|
||||
case store.ScopeKey, store.ScopeUser:
|
||||
return store.Scope(kind), what, nil
|
||||
case store.ScopeOrg, store.ScopeTeam, store.ScopeProject:
|
||||
if a.quotas == nil {
|
||||
return store.Scope(kind), what, nil
|
||||
}
|
||||
scope, id, _, err := a.resolveScope(ctx, target)
|
||||
return scope, id, err
|
||||
}
|
||||
return "", "", fmt.Errorf("%q: org, team, project, key or user", kind)
|
||||
}
|
||||
|
||||
// kbFind reads a collection by ID, by its name when that is unique, or by
|
||||
// ORG/NAME.
|
||||
func (a *API) kbFind(ctx context.Context, ref string) (store.KBCollection, error) {
|
||||
ref = strings.TrimSpace(ref)
|
||||
if c, err := a.kb.Store.Collection(ctx, ref); err == nil {
|
||||
return c, nil
|
||||
}
|
||||
cs, err := a.kb.Store.Collections(ctx)
|
||||
if err != nil {
|
||||
return store.KBCollection{}, err
|
||||
}
|
||||
names := a.names(ctx)
|
||||
org, name, scoped := strings.Cut(ref, "/")
|
||||
var found []store.KBCollection
|
||||
for _, c := range cs {
|
||||
switch {
|
||||
case scoped && strings.EqualFold(c.Name, name) &&
|
||||
(c.OrgID == org || strings.EqualFold(names["org:"+c.OrgID], org)):
|
||||
found = append(found, c)
|
||||
case !scoped && strings.EqualFold(c.Name, ref):
|
||||
found = append(found, c)
|
||||
}
|
||||
}
|
||||
switch len(found) {
|
||||
case 0:
|
||||
return store.KBCollection{}, fmt.Errorf("no collection %q", ref)
|
||||
case 1:
|
||||
return found[0], nil
|
||||
}
|
||||
return store.KBCollection{}, fmt.Errorf("%q names a collection in more than one organization; write ORG/%s", ref, ref)
|
||||
}
|
||||
|
||||
func (a *API) kbCreate(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.kbReady(w) {
|
||||
return
|
||||
}
|
||||
var in KBCreateRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
scope, id, err := a.kbTarget(r.Context(), in.Owner)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
org := ""
|
||||
if scope == store.ScopeUser {
|
||||
if in.Org == "" {
|
||||
writeError(w, http.StatusBadRequest, "a person's collection names its organization (org)")
|
||||
return
|
||||
}
|
||||
if _, org, err = a.kbTarget(r.Context(), "org:"+in.Org); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
m, ok := a.ctl.Catalog().Resolve(in.Model)
|
||||
if !ok {
|
||||
writeError(w, http.StatusBadRequest, "no model "+in.Model+" in the catalog")
|
||||
return
|
||||
}
|
||||
if !m.Capabilities.Embeddings {
|
||||
writeError(w, http.StatusBadRequest, m.Name+" does not embed (capabilities.embeddings); a collection needs an embedding model")
|
||||
return
|
||||
}
|
||||
dims := in.Dims
|
||||
if dims == 0 {
|
||||
if a.kb.Embed == nil {
|
||||
writeError(w, http.StatusBadRequest, "give the model's dimension (dims): nothing here can ask it")
|
||||
return
|
||||
}
|
||||
// Asked of the model: a dimension typed by hand and wrong would be
|
||||
// found only when the first document is refused.
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Minute)
|
||||
defer cancel()
|
||||
v, err := a.kb.Embed(ctx, m.Name, "llmgw")
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadGateway, "asking "+m.Name+" for its dimension: "+err.Error()+
|
||||
" (is it running? or give dims)")
|
||||
return
|
||||
}
|
||||
dims = len(v)
|
||||
}
|
||||
c, err := a.kb.Store.CreateCollection(r.Context(), store.KBCollection{OwnerScope: scope, OwnerID: id, OrgID: org,
|
||||
Name: in.Name, EmbedModel: m.Name, Dims: dims, CreatedBy: actorOf(r)})
|
||||
if err != nil {
|
||||
code := http.StatusBadRequest
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
code = http.StatusNotFound
|
||||
}
|
||||
writeError(w, code, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, KBCollectionView{ID: c.ID, Name: c.Name, Org: c.OrgID,
|
||||
Owner: string(c.OwnerScope) + ":" + c.OwnerID, EmbedModel: c.EmbedModel, Dims: c.Dims,
|
||||
Grants: []KBGrantView{}, CreatedAt: c.CreatedAt, CreatedBy: c.CreatedBy})
|
||||
}
|
||||
|
||||
func (a *API) kbCollection(w http.ResponseWriter, r *http.Request) (store.KBCollection, bool) {
|
||||
if !a.kbReady(w) {
|
||||
return store.KBCollection{}, false
|
||||
}
|
||||
c, err := a.kbFind(r.Context(), r.PathValue("id"))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return store.KBCollection{}, false
|
||||
}
|
||||
return c, true
|
||||
}
|
||||
|
||||
func (a *API) kbDelete(w http.ResponseWriter, r *http.Request) {
|
||||
c, ok := a.kbCollection(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := a.kb.Store.DeleteCollection(r.Context(), c.ID); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"id": c.ID, "name": c.Name, "deleted": true})
|
||||
}
|
||||
|
||||
func (a *API) kbDocuments(w http.ResponseWriter, r *http.Request) {
|
||||
c, ok := a.kbCollection(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
docs, err := a.kb.Store.Documents(r.Context(), c.ID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
out := []KBDocumentView{}
|
||||
for _, d := range docs {
|
||||
out = append(out, KBDocumentView{Name: d.Name, Digest: d.Digest, Bytes: d.Bytes, Chunks: d.Chunks,
|
||||
AddedAt: d.AddedAt, AddedBy: d.AddedBy})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
func (a *API) kbDeleteDocument(w http.ResponseWriter, r *http.Request) {
|
||||
c, ok := a.kbCollection(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
name := r.URL.Query().Get("name")
|
||||
gone, err := a.kb.Store.DeleteDocument(r.Context(), c.ID, name)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if !gone {
|
||||
writeError(w, http.StatusNotFound, "no document "+name+" in "+c.Name)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"name": name, "deleted": true})
|
||||
}
|
||||
|
||||
func (a *API) kbGrant(grant bool) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.kbReady(w) {
|
||||
return
|
||||
}
|
||||
var in KBGrantRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
c, err := a.kbFind(r.Context(), in.Collection)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
scope, id, err := a.kbTarget(r.Context(), in.Target)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if !grant {
|
||||
gone, err := a.kb.Store.Revoke(r.Context(), scope, id, c.ID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if !gone {
|
||||
writeError(w, http.StatusNotFound, "no such grant")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, KBGrantView{Target: string(scope) + ":" + id})
|
||||
return
|
||||
}
|
||||
perm := store.KBPerm(strings.ToLower(strings.TrimSpace(in.Perm)))
|
||||
if perm == "" {
|
||||
perm = store.KBRead
|
||||
}
|
||||
if err := a.kb.Store.Grant(r.Context(), store.KBGrant{Scope: scope, ScopeID: id, Collection: c.ID,
|
||||
Perm: perm, AddedBy: actorOf(r)}); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, KBGrantView{Target: string(scope) + ":" + id, Perm: string(perm), At: time.Now(), By: actorOf(r)})
|
||||
}
|
||||
}
|
||||
|
||||
func (a *API) kbSearch(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.kbReady(w) {
|
||||
return
|
||||
}
|
||||
var in KBSearchRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
c, err := a.kbFind(r.Context(), in.Collection)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
k := in.K
|
||||
if k <= 0 || k > 50 {
|
||||
k = 5
|
||||
}
|
||||
q := store.KBSearch{OrgID: c.OrgID, Collections: []store.KBCollection{c}, Text: in.Query, K: k}
|
||||
// Said, not swallowed: a search that quietly lost its meaning half
|
||||
// looks like a collection that does not know the answer.
|
||||
if a.kb.Embed != nil {
|
||||
v, err := a.kb.Embed(r.Context(), c.EmbedModel, in.Query)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadGateway, c.EmbedModel+" did not embed the question: "+err.Error())
|
||||
return
|
||||
}
|
||||
q.Embeddings = map[string][]float32{c.EmbedModel: v}
|
||||
}
|
||||
hits, err := a.kb.Store.SearchChunks(r.Context(), q)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
out := []KBHitView{}
|
||||
for _, h := range hits {
|
||||
out = append(out, KBHitView{Document: h.Document, Ord: h.Ord, Score: h.Score, Text: h.Body})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
func (a *API) kbSetDefault(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.kbReady(w) {
|
||||
return
|
||||
}
|
||||
var in KBDefaultRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
scope, id, err := a.kbTarget(r.Context(), in.Target)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
d := store.KBDefault{Scope: scope, ScopeID: id, AddedBy: actorOf(r)}
|
||||
for _, ref := range in.Collections {
|
||||
c, err := a.kbFind(r.Context(), ref)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
d.Collections = append(d.Collections, c.ID)
|
||||
}
|
||||
if err := a.kb.Store.SetKBDefault(r.Context(), d); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, KBDefaultView{Target: string(scope) + ":" + id, Collections: in.Collections, At: time.Now(), By: actorOf(r)})
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func kbModels() []catalog.Model {
|
||||
return append(testModels(), catalog.Model{Name: "bge-m3", Source: "/e",
|
||||
Capabilities: catalog.Capabilities{Embeddings: true},
|
||||
Profiles: []catalog.Profile{{Name: "cpu", GPUClass: "h100-80g", TP: 1}}})
|
||||
}
|
||||
|
||||
func TestKnowledgeBasesListSaysWhyThereAreNone(t *testing.T) {
|
||||
a, _ := setup(t, kbModels())
|
||||
s, err := store.OpenSQLite(context.Background(), filepath.Join(t.TempDir(), "s.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
a.SetKB(&KBAdmin{Store: s})
|
||||
var v KBView
|
||||
if code := call(t, a, "GET", "/api/v1/kb", "", &v); code != http.StatusOK || v.Available || !strings.Contains(v.Reason, "SQLite") {
|
||||
t.Errorf("got %d %+v", code, v)
|
||||
}
|
||||
if code := call(t, a, "POST", "/api/v1/kb/collections", `{"owner":"org:x","name":"n","model":"bge-m3"}`, nil); code != http.StatusNotImplemented {
|
||||
t.Errorf("creating on SQLite: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKnowledgeBasesAdministered(t *testing.T) {
|
||||
dsn := os.Getenv("LLMGW_TEST_POSTGRES")
|
||||
if dsn == "" {
|
||||
t.Skip("set LLMGW_TEST_POSTGRES to a pgvector database to run this")
|
||||
}
|
||||
if u, err := url.Parse(dsn); err != nil || !strings.Contains(u.Path, "test") {
|
||||
t.Fatalf("refusing a database whose name does not say test")
|
||||
}
|
||||
ctx := context.Background()
|
||||
s, err := store.OpenPostgres(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.KnowledgeAvailable(); err != nil {
|
||||
t.Skip(err)
|
||||
}
|
||||
a, _ := setup(t, kbModels())
|
||||
a.SetQuotas(s)
|
||||
probes := 0
|
||||
a.SetKB(&KBAdmin{Store: s, Embed: func(_ context.Context, model, _ string) ([]float32, error) {
|
||||
probes++
|
||||
return make([]float32, 4), nil
|
||||
}})
|
||||
sfx := strings.ToLower(store.NewID(""))
|
||||
org, _ := s.CreateOrg(ctx, "Acme"+sfx)
|
||||
team, _ := s.CreateTeam(ctx, org, "hr")
|
||||
_, _ = s.CreateProject(ctx, team, "docs")
|
||||
other, _ := s.CreateOrg(ctx, "Other"+sfx)
|
||||
|
||||
// Owner by path; the dimension asked of the model.
|
||||
var c KBCollectionView
|
||||
rec := func(method, path, body string, out any) int { return call(t, a, method, path, body, out) }
|
||||
body := `{"owner":"team:Acme` + sfx + `/hr","name":"politikalar","model":"bge-m3"}`
|
||||
if code := callCreated(t, a, body, &c); code != http.StatusCreated || c.Dims != 4 || probes != 1 || c.Org != org {
|
||||
t.Fatalf("create: %d %+v (probes %d)", code, c, probes)
|
||||
}
|
||||
// A model that does not embed is refused before anything is written.
|
||||
if code := rec("POST", "/api/v1/kb/collections", `{"owner":"org:`+org+`","name":"x","model":"llama"}`, nil); code != http.StatusBadRequest {
|
||||
t.Errorf("a chat model as the embedder: %d", code)
|
||||
}
|
||||
// Grants by path, never across organizations.
|
||||
if code := rec("POST", "/api/v1/kb/grants", `{"collection":"Acme`+sfx+`/politikalar","target":"project:Acme`+sfx+`/hr/docs","perm":"write"}`, nil); code != http.StatusOK {
|
||||
t.Errorf("grant: %d", code)
|
||||
}
|
||||
if code := rec("POST", "/api/v1/kb/grants", `{"collection":"`+c.ID+`","target":"org:`+other+`"}`, nil); code != http.StatusBadRequest {
|
||||
t.Errorf("a grant to another organization: %d", code)
|
||||
}
|
||||
var v KBView
|
||||
rec("GET", "/api/v1/kb", "", &v)
|
||||
var got *KBCollectionView
|
||||
for i := range v.Collections {
|
||||
if v.Collections[i].ID == c.ID {
|
||||
got = &v.Collections[i]
|
||||
}
|
||||
}
|
||||
if got == nil || len(got.Grants) != 1 || got.Grants[0].Perm != "write" || !strings.Contains(got.OwnerName, "hr") {
|
||||
t.Fatalf("list: %+v", got)
|
||||
}
|
||||
// By ORG/NAME too.
|
||||
if code := rec("DELETE", "/api/v1/kb/grants", `{"collection":"Acme`+sfx+`/politikalar","target":"project:Acme`+sfx+`/hr/docs"}`, nil); code != http.StatusOK {
|
||||
t.Errorf("revoke: %d", code)
|
||||
}
|
||||
if code := rec("DELETE", "/api/v1/kb/collections/"+c.ID, "", nil); code != http.StatusOK {
|
||||
t.Errorf("delete: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func callCreated(t *testing.T, a *API, body string, out any) int {
|
||||
t.Helper()
|
||||
mux := http.NewServeMux()
|
||||
a.Register(mux)
|
||||
r, _ := http.NewRequest("POST", "/api/v1/kb/collections", strings.NewReader(body))
|
||||
r.Header.Set("Content-Type", "application/json")
|
||||
w := &recorder{h: http.Header{}}
|
||||
mux.ServeHTTP(w, r)
|
||||
if w.code == http.StatusCreated {
|
||||
_ = json.Unmarshal(w.body, out)
|
||||
} else {
|
||||
t.Logf("create answered %d %s", w.code, w.body)
|
||||
}
|
||||
return w.code
|
||||
}
|
||||
|
||||
type recorder struct {
|
||||
h http.Header
|
||||
code int
|
||||
body []byte
|
||||
}
|
||||
|
||||
func (r *recorder) Header() http.Header { return r.h }
|
||||
func (r *recorder) Write(b []byte) (int, error) {
|
||||
if r.code == 0 {
|
||||
r.code = http.StatusOK
|
||||
}
|
||||
r.body = append(r.body, b...)
|
||||
return len(b), nil
|
||||
}
|
||||
func (r *recorder) WriteHeader(c int) { r.code = c }
|
||||
@@ -66,6 +66,7 @@ var PathParams = map[string]string{
|
||||
"quotas/{scope}": "The scope: org:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT, or their IDs after the colon.",
|
||||
"federation/{peer}": "The federation peer's name, as `llmgw ctl federation list` lists it.",
|
||||
"holds/{domain}": "The domain as level:name, rack:r12-07, as `llmgw ctl facility` lists it.",
|
||||
"collections/{id}": "The collection's ID, its name when that is unique, or ORG/NAME.",
|
||||
}
|
||||
|
||||
// Routes is every endpoint Register mounts.
|
||||
@@ -254,11 +255,46 @@ var Routes = []Route{
|
||||
{Name: "since", Description: "How far back: 90m, 24h, 7d; 24h when absent."},
|
||||
},
|
||||
Response: GuardEventsView{}},
|
||||
{Method: "GET", Path: "/api/v1/guard/judges", Summary: "The judge models the policies ask, and what is wrong with each",
|
||||
Response: []JudgeView{}},
|
||||
{Method: "POST", Path: "/api/v1/guard/judges/ask", Summary: "Have a judge read a text and say what it answered; nothing is kept",
|
||||
Request: JudgeAskRequest{}, Response: JudgeAskView{}},
|
||||
{Method: "POST", Path: "/api/v1/guard/attachments", Summary: "Attach a guardrail policy",
|
||||
Request: GuardAttachmentRequest{}, Response: guard.Attachment{}},
|
||||
{Method: "DELETE", Path: "/api/v1/guard/attachments", Summary: "Detach a guardrail policy",
|
||||
Request: GuardAttachmentRequest{}, Response: guard.Attachment{}},
|
||||
|
||||
// Knowledge bases: collections of documents, which people fill with
|
||||
// llmgw-kb from their own machines through the gateway.
|
||||
{Method: "GET", Path: "/api/v1/kb", Summary: "Every knowledge-base collection, with its documents' count and grants",
|
||||
Description: "Says why there are none when the store cannot hold them (SQLite, or PostgreSQL without pgvector).",
|
||||
Response: KBView{}},
|
||||
{Method: "POST", Path: "/api/v1/kb/collections", Summary: "Make a collection, owned by an organization, team, project or person",
|
||||
Description: "The embedding model's dimension is asked of the model when dims is not given, so the model must be able to serve.",
|
||||
Request: KBCreateRequest{}, Response: KBCollectionView{}, Status: http.StatusCreated},
|
||||
{Method: "DELETE", Path: "/api/v1/kb/collections/{id}", Summary: "Remove a collection with its documents and grants",
|
||||
Response: map[string]any{}},
|
||||
{Method: "GET", Path: "/api/v1/kb/collections/{id}/documents", Summary: "A collection's documents",
|
||||
Response: []KBDocumentView{}},
|
||||
{Method: "DELETE", Path: "/api/v1/kb/collections/{id}/documents", Summary: "Remove one document from a collection",
|
||||
Query: []Param{{Name: "name", Description: "The document's name, as listed.", Required: true}},
|
||||
Response: map[string]any{}},
|
||||
{Method: "POST", Path: "/api/v1/kb/grants", Summary: "Let a scope, and everything below it, read or write a collection",
|
||||
Description: "Never outside the collection's organization.",
|
||||
Request: KBGrantRequest{}, Response: KBGrantView{}},
|
||||
{Method: "DELETE", Path: "/api/v1/kb/grants", Summary: "Take a grant away",
|
||||
Request: KBGrantRequest{}, Response: KBGrantView{}},
|
||||
{Method: "POST", Path: "/api/v1/kb/defaults", Summary: "Set the collections a part of the tree answers from when a request names none",
|
||||
Description: "On an organization, team, project or key; the narrowest one with a default wins. No collections clears it. " +
|
||||
"A request's own X-Llmgw-Knowledge overrides it, and X-Llmgw-Knowledge: none turns it off.",
|
||||
Request: KBDefaultRequest{}, Response: KBDefaultView{}},
|
||||
{Method: "POST", Path: "/api/v1/kb/search", Summary: "Try a search of a collection, as an administrator",
|
||||
Description: "No tenancy applies: this is for checking what a collection holds and finds.",
|
||||
Request: KBSearchRequest{}, Response: []KBHitView{}},
|
||||
|
||||
{Method: "POST", Path: "/api/v1/agents/restart", Summary: "Have every node's agent start again; their engines keep serving",
|
||||
Response: AgentRestartView{}},
|
||||
|
||||
// The cluster's settings: everything in server.yaml that is not one
|
||||
// machine's own, the same on every control plane.
|
||||
{Method: "GET", Path: "/api/v1/settings", Summary: "The cluster's settings in force, or one earlier version",
|
||||
|
||||
@@ -42,6 +42,14 @@ type Summary struct {
|
||||
// ControlPlanes is the cluster's control planes when there is more than
|
||||
// one: which leads, which stand by, and the floating address in front.
|
||||
ControlPlanes ControlPlanes `json:"control_planes,omitzero"`
|
||||
// Judges are the guardrails' judge models with something wrong.
|
||||
Judges []JudgeView `json:"judges,omitempty"`
|
||||
// Stopped says the operators have stopped the service (llmgw stop): no
|
||||
// model runs, and requests for one are answered 503.
|
||||
Stopped bool `json:"stopped,omitempty"`
|
||||
// GuardOff says the guardrails are turned off (guard.enabled: false):
|
||||
// no request is checked, whatever the policies say.
|
||||
GuardOff bool `json:"guard_off,omitempty"`
|
||||
}
|
||||
|
||||
// StoreMember is one member of a replicated store, as its Patroni reports it.
|
||||
@@ -148,7 +156,15 @@ const idleThreshold = 10 * time.Minute
|
||||
|
||||
// Summary builds the fleet summary.
|
||||
func (a *API) Summary() Summary {
|
||||
s := Summary{Version: version.Version}
|
||||
s := Summary{Version: version.Version, Stopped: a.ctl.Reconciler().Stopped()}
|
||||
if a.guard != nil && a.guard.Library != nil && !a.guard.Library().On() {
|
||||
s.GuardOff = true
|
||||
}
|
||||
for _, j := range a.Judges() {
|
||||
if j.Problem != "" {
|
||||
s.Judges = append(s.Judges, j)
|
||||
}
|
||||
}
|
||||
if a.storeStatus != nil {
|
||||
members, err := a.storeStatus()
|
||||
s.Store = members
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// AgentRestartView names the nodes whose agents were told to restart.
|
||||
type AgentRestartView struct {
|
||||
Nodes []string `json:"nodes"`
|
||||
}
|
||||
|
||||
// SetAgentRestarter enables POST /api/v1/agents/restart.
|
||||
func (a *API) SetAgentRestarter(f func(reason string) []string) { a.restartAgents = f }
|
||||
|
||||
func (a *API) restartAgentsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if a.restartAgents == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no agents to restart")
|
||||
return
|
||||
}
|
||||
nodes := a.restartAgents("llmgw restart, by " + actorOf(r))
|
||||
writeJSON(w, http.StatusAccepted, AgentRestartView{Nodes: nodes})
|
||||
}
|
||||
@@ -169,6 +169,10 @@ func AdoptedFromRegister(nodeID string, shards []*llmgwv1.RunningShard) []reconc
|
||||
Fabric: s.GetFabric(),
|
||||
ProfileFP: s.GetProfileFp(),
|
||||
}
|
||||
if c := s.GetCompute(); c != nil {
|
||||
a.Compute = &reconciler.Compute{Backend: c.GetBackend(), Devices: c.GetDevices(),
|
||||
Offloaded: int(c.GetLayersOffloaded()), Total: int(c.GetLayersTotal())}
|
||||
}
|
||||
if a.Rank == 0 && s.GetLeaderAddr() != "" {
|
||||
a.Endpoint = fmt.Sprintf("http://%s:%d", s.GetLeaderAddr(), s.GetHttpPort())
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package agentsvc
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
)
|
||||
|
||||
@@ -40,3 +42,20 @@ func (s *Service) BroadcastFirewall() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RestartAgents has every connected agent start again, one at a time a
|
||||
// moment apart, and names those it told. Their engines keep serving.
|
||||
func (s *Service) RestartAgents(reason string) []string {
|
||||
msg := &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_RestartAgent{
|
||||
RestartAgent: &llmgwv1.RestartAgent{Reason: reason}}}
|
||||
var told []string
|
||||
for _, node := range s.ConnectedNodes() {
|
||||
if err := s.send(node, msg); err != nil {
|
||||
s.log.Warn("could not tell a node to restart its agent", "node", node, "err", err)
|
||||
continue
|
||||
}
|
||||
told = append(told, node)
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
return told
|
||||
}
|
||||
@@ -36,6 +36,8 @@ type StatusSink interface {
|
||||
// still starting, and treating each report as a state change would reset
|
||||
// the timers that decide when a slow start has become a failed one.
|
||||
OnFetchProgress(replicaID, shardID, source string, done, total int64)
|
||||
// OnShardCompute records where a shard's engine said it runs the model.
|
||||
OnShardCompute(replicaID, shardID string, c reconciler.Compute)
|
||||
Adopt(nodeID string, shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard)
|
||||
}
|
||||
|
||||
@@ -447,6 +449,16 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
|
||||
f.GetSource(), f.GetBytesDone(), f.GetBytesTotal())
|
||||
return
|
||||
}
|
||||
if c := m.Shard.GetCompute(); c != nil {
|
||||
s.status.OnShardCompute(m.Shard.GetReplicaId(), m.Shard.GetShardId(), reconciler.Compute{
|
||||
Backend: c.GetBackend(), Devices: c.GetDevices(),
|
||||
Offloaded: int(c.GetLayersOffloaded()), Total: int(c.GetLayersTotal()),
|
||||
})
|
||||
// Compute alone, with no state: not a transition.
|
||||
if m.Shard.GetState() == llmgwv1.ShardStatus_STATE_UNSPECIFIED {
|
||||
return
|
||||
}
|
||||
}
|
||||
st := StateFromWire(m.Shard.GetState())
|
||||
s.status.OnShardStatus(m.Shard.GetReplicaId(), m.Shard.GetShardId(), st, m.Shard.GetError())
|
||||
if m.Shard.GetState() == llmgwv1.ShardStatus_STATE_FAILED {
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
// Scaling is how many replicas a model should have.
|
||||
@@ -164,9 +165,13 @@ type SLO struct {
|
||||
|
||||
// Profile is how a model runs on one GPU class.
|
||||
type Profile struct {
|
||||
Name string `json:"name,omitempty" koanf:"name"`
|
||||
Engine string `json:"engine,omitempty" koanf:"engine"`
|
||||
GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"`
|
||||
Name string `json:"name,omitempty" koanf:"name"`
|
||||
Engine string `json:"engine,omitempty" koanf:"engine"`
|
||||
// EngineVersion restricts the profile to nodes whose installed engine's
|
||||
// version starts with it: "0.11" takes vLLM 0.11.0 and 0.11.1, not
|
||||
// 0.10.2. Empty takes whatever version the node has.
|
||||
EngineVersion string `json:"engine_version,omitempty" koanf:"engine_version"`
|
||||
GPUClass string `json:"gpu_class,omitempty" koanf:"gpu_class"`
|
||||
|
||||
TP int `json:"tp,omitempty" koanf:"tp"`
|
||||
PP int `json:"pp,omitempty" koanf:"pp"`
|
||||
@@ -382,9 +387,11 @@ func (m Model) IsGuard() bool { return m.Role == RoleGuard }
|
||||
|
||||
// GuardModel is how a judge model's answers are read.
|
||||
type GuardModel struct {
|
||||
// Format is llamaguard3 (Meta's Llama Guard 3: "safe", or "unsafe" and
|
||||
// the categories), shieldgemma (Google's ShieldGemma: Yes or No), or
|
||||
// json ({"safe": false, "categories": ["..."]}).
|
||||
// Format is qwen3guard (Qwen3Guard-Gen: "Safety: Safe", "Unsafe" or
|
||||
// "Controversial", and the categories), llamaguard3 (Meta's Llama Guard
|
||||
// 3: "safe", or "unsafe" and the categories), shieldgemma (Google's
|
||||
// ShieldGemma: Yes or No), or json ({"safe": false, "categories":
|
||||
// ["..."]}).
|
||||
Format string `json:"format" koanf:"format"`
|
||||
// Categories the model is asked about, for shieldgemma — each a policy
|
||||
// sentence — and the names reported for llamaguard3's S1, S2, ….
|
||||
@@ -704,12 +711,10 @@ func (m Model) validate() error {
|
||||
}
|
||||
case RoleGuard:
|
||||
if m.Guard == nil {
|
||||
return fmt.Errorf("model %q: a guard model needs guard.format: llamaguard3, shieldgemma or json", m.Name)
|
||||
return fmt.Errorf("model %q: a guard model needs guard.format: %s", m.Name, strings.Join(guard.Formats, ", "))
|
||||
}
|
||||
switch m.Guard.Format {
|
||||
case "llamaguard3", "shieldgemma", "json":
|
||||
default:
|
||||
return fmt.Errorf("model %q: guard.format %q is not llamaguard3, shieldgemma or json", m.Name, m.Guard.Format)
|
||||
if !guard.KnownFormat(m.Guard.Format) {
|
||||
return fmt.Errorf("model %q: guard.format %q is not one of %s", m.Name, m.Guard.Format, strings.Join(guard.Formats, ", "))
|
||||
}
|
||||
if m.Scaling.OnDemand() {
|
||||
// A judge loaded on demand makes the first request of every
|
||||
@@ -885,18 +890,19 @@ func (m Model) PlacementProfiles() []placement.Profile {
|
||||
out := make([]placement.Profile, 0, len(m.Profiles))
|
||||
for _, p := range m.Profiles {
|
||||
out = append(out, placement.Profile{
|
||||
Name: p.Name,
|
||||
Engine: p.Engine,
|
||||
GPUClass: p.GPUClass,
|
||||
GPUsPerNode: p.GPUsPerNode,
|
||||
Nodes: p.Nodes,
|
||||
Priority: p.Priority,
|
||||
NodeSelector: p.NodeSelector,
|
||||
MinMemoryGB: p.MinMemoryGB,
|
||||
MinDiskGB: p.MinDiskGB,
|
||||
CPUCores: p.CPUCores,
|
||||
PowerWatts: p.PowerWatts,
|
||||
KeepWithin: facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))),
|
||||
Name: p.Name,
|
||||
Engine: p.Engine,
|
||||
EngineVersion: p.EngineVersion,
|
||||
GPUClass: p.GPUClass,
|
||||
GPUsPerNode: p.GPUsPerNode,
|
||||
Nodes: p.Nodes,
|
||||
Priority: p.Priority,
|
||||
NodeSelector: p.NodeSelector,
|
||||
MinMemoryGB: p.MinMemoryGB,
|
||||
MinDiskGB: p.MinDiskGB,
|
||||
CPUCores: p.CPUCores,
|
||||
PowerWatts: p.PowerWatts,
|
||||
KeepWithin: facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))),
|
||||
// The model's source, so placement can prefer nodes that already
|
||||
// hold the weights rather than one that must first move them.
|
||||
Source: m.Source,
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
@@ -137,8 +138,12 @@ type Request struct {
|
||||
// Replicas to run; 0 makes it on demand, loaded when first asked for.
|
||||
Replicas int `json:"replicas"`
|
||||
// GuardFormat makes it a judge model for the guardrails (role: guard):
|
||||
// llamaguard3, shieldgemma or json. Empty serves clients.
|
||||
// llamaguard3, qwen3guard, shieldgemma or json. Empty serves clients.
|
||||
GuardFormat string `json:"guard_format,omitempty"`
|
||||
// Class is what it may take from others and they from it when there is
|
||||
// not room for both: critical, standard or opportunistic. Empty is the
|
||||
// catalog's default for its role.
|
||||
Class string `json:"class,omitempty"`
|
||||
// CtxPool is llama.cpp's --ctx-size; ContextLength what one request may
|
||||
// use. They default to 16384 and 4096, what four parallel slots divide;
|
||||
// llama.cpp is given as many slots as the pool holds requests.
|
||||
@@ -252,10 +257,13 @@ func (s *Server) Start(req Request) (Job, error) {
|
||||
default:
|
||||
return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format)
|
||||
}
|
||||
switch req.GuardFormat {
|
||||
case "", "llamaguard3", "shieldgemma", "json":
|
||||
if req.GuardFormat != "" && !guard.KnownFormat(req.GuardFormat) {
|
||||
return Job{}, fmt.Errorf("the guard format is one of %s, not %q", strings.Join(guard.Formats, ", "), req.GuardFormat)
|
||||
}
|
||||
switch req.Class {
|
||||
case "", catalog.ClassCritical, catalog.ClassStandard, catalog.ClassOpportunistic:
|
||||
default:
|
||||
return Job{}, fmt.Errorf("the guard format is llamaguard3, shieldgemma or json, not %q", req.GuardFormat)
|
||||
return Job{}, fmt.Errorf("the class is critical, standard or opportunistic, not %q", req.Class)
|
||||
}
|
||||
if req.GuardFormat != "" && req.Replicas == 0 {
|
||||
return Job{}, errors.New("a judge model must always run: give it --replicas 1 or more, not --on-demand")
|
||||
@@ -618,6 +626,7 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
|
||||
if req.GuardFormat != "" {
|
||||
m.Role, m.Guard, m.RoutingProfile = catalog.RoleGuard, &catalog.GuardModel{Format: req.GuardFormat}, ""
|
||||
}
|
||||
m.Class = req.Class
|
||||
s.step(j, "catalog", Running, "adding "+name)
|
||||
if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil {
|
||||
s.fail(j, "catalog", err)
|
||||
|
||||
@@ -25,9 +25,12 @@ import (
|
||||
|
||||
// Profile is the subset of a catalog profile placement needs.
|
||||
type Profile struct {
|
||||
Name string
|
||||
Engine string
|
||||
GPUClass string
|
||||
Name string
|
||||
Engine string
|
||||
// EngineVersion is a prefix the node's installed engine version must
|
||||
// have; empty takes any.
|
||||
EngineVersion string
|
||||
GPUClass string
|
||||
// GPUsPerNode is how many GPUs each node contributes.
|
||||
GPUsPerNode int
|
||||
// Nodes is how many nodes the replica spans. Above 1, a fabric is required.
|
||||
@@ -614,8 +617,13 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
|
||||
// and either way not a place to start an engine. sim is built into
|
||||
// the agent and never installed.
|
||||
if p.Engine != "" && p.Engine != "sim" && (n.EnginesProbed || len(n.EngineVersions) > 0) {
|
||||
if _, ok := n.EngineVersions[p.Engine]; !ok {
|
||||
reject("does not have " + p.Engine + " installed" + installing(n, p.Engine))
|
||||
v, ok := n.EngineVersions[p.Engine]
|
||||
if !ok {
|
||||
reject("does not have " + p.Engine + " installed" + installing(n, p.Engine) + hasEngines(n))
|
||||
continue
|
||||
}
|
||||
if p.EngineVersion != "" && !strings.HasPrefix(v, p.EngineVersion) {
|
||||
reject(fmt.Sprintf("has %s %s, not the %s the profile asks for", p.Engine, v, p.EngineVersion))
|
||||
continue
|
||||
}
|
||||
}
|
||||
@@ -828,6 +836,19 @@ func (s *Scheduler) claimedCores(nodeID string) int {
|
||||
}
|
||||
|
||||
// installing says why an engine is missing, when the node has said.
|
||||
// hasEngines is what a node does have, for a rejection over what it does not.
|
||||
func hasEngines(n inventory.Node) string {
|
||||
if len(n.EngineVersions) == 0 {
|
||||
return ""
|
||||
}
|
||||
var parts []string
|
||||
for e, v := range n.EngineVersions {
|
||||
parts = append(parts, e+" "+v)
|
||||
}
|
||||
slices.Sort(parts)
|
||||
return "; it has " + strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
func installing(n inventory.Node, engine string) string {
|
||||
for _, p := range n.Provision {
|
||||
if p.Engine == engine && p.Detail != "" {
|
||||
|
||||
@@ -1071,3 +1071,31 @@ func TestANodeWithoutIMEXIsNotCountedInItsDomain(t *testing.T) {
|
||||
t.Errorf("the explanation should say why the domain was passed over:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProfileCanAskForAnEngineVersion(t *testing.T) {
|
||||
old := withEngines("volta", 8, "vllm", "llamacpp")
|
||||
old.EngineVersions["vllm"] = "0.10.2"
|
||||
newer := withEngines("hopper", 8, "vllm")
|
||||
newer.EngineVersions["vllm"] = "0.11.1"
|
||||
s := New(store(old, newer))
|
||||
|
||||
res := s.Place([]Profile{{Name: "tp1", Engine: "vllm", EngineVersion: "0.11", GPUClass: "h100-80g",
|
||||
GPUsPerNode: 1, Nodes: 1}}, nil)
|
||||
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "hopper" {
|
||||
t.Fatalf("placed where the version is not:\n%s", res.Explain())
|
||||
}
|
||||
|
||||
only := New(store(old))
|
||||
res = only.Place([]Profile{{Name: "tp1", Engine: "vllm", EngineVersion: "0.11", GPUClass: "h100-80g",
|
||||
GPUsPerNode: 1, Nodes: 1}}, nil)
|
||||
if res.Placement != nil || !strings.Contains(res.Explain(), "has vllm 0.10.2, not the 0.11") {
|
||||
t.Errorf("the rejection should say which version is there:\n%s", res.Explain())
|
||||
}
|
||||
|
||||
// A node without the engine says what it has instead.
|
||||
res = New(store(withEngines("cpu-only", 8, "llamacpp"))).Place([]Profile{{Name: "tp1", Engine: "vllm",
|
||||
GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1}}, nil)
|
||||
if !strings.Contains(res.Explain(), "it has llamacpp") {
|
||||
t.Errorf("the rejection should say what the node has:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
@@ -130,13 +130,19 @@ func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) {
|
||||
}
|
||||
for _, s := range group {
|
||||
if _, dup := have[s.ShardID]; dup {
|
||||
// Known already, but what it computes on may not be.
|
||||
for i := range rep.Shards {
|
||||
if rep.Shards[i].ID == s.ShardID && rep.Shards[i].Compute == nil {
|
||||
rep.Shards[i].Compute = s.Compute
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
have[s.ShardID] = struct{}{}
|
||||
rep.Shards = append(rep.Shards, Shard{
|
||||
ID: s.ShardID, NodeID: s.NodeID, Rank: s.Rank,
|
||||
GPUIndices: s.GPUIndices, HCAs: s.HCAs,
|
||||
HTTPPort: s.HTTPPort, State: s.State,
|
||||
HTTPPort: s.HTTPPort, State: s.State, Compute: s.Compute,
|
||||
})
|
||||
// Reclaim the GPUs and the memory, or the next placement hands them
|
||||
// out twice. The memory comes from the catalog rather than the agent:
|
||||
@@ -254,6 +260,9 @@ type AdoptedShard struct {
|
||||
ProfileFP string
|
||||
// Endpoint is the leader's base URL, set on the rank 0 shard.
|
||||
Endpoint string
|
||||
// Compute is where the engine said it put the model, when the agent
|
||||
// kept that.
|
||||
Compute *Compute
|
||||
}
|
||||
|
||||
// adoptedCores is what an adopted shard is taken to hold of its node's CPUs.
|
||||
|
||||
@@ -81,6 +81,10 @@ func (r *Reconciler) Want(model string) bool {
|
||||
// feature: a cluster with more models than GPUs can hold every model in its
|
||||
// catalog and only the wanted ones in its cards.
|
||||
func (r *Reconciler) desiredFor(m catalog.Model) int {
|
||||
// Stopped by the operators: nothing runs, whatever the catalog asks.
|
||||
if r.stopped.Load() {
|
||||
return 0
|
||||
}
|
||||
base := r.baseFor(m)
|
||||
if base == 0 {
|
||||
return 0
|
||||
|
||||
@@ -46,3 +46,36 @@ func TestWithoutLeadershipConfiguredNothingChanges(t *testing.T) {
|
||||
t.Error("a reconciler with no leadership configured placed nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// switchLeader is leadership that can change hands.
|
||||
type switchLeader struct{ on bool }
|
||||
|
||||
func (s *switchLeader) IsLeader() bool { return s.on }
|
||||
|
||||
func TestAStandbyThatTakesOverWaitsForTheFleetBeforePlacing(t *testing.T) {
|
||||
r, inv, disp, c := settlingHarness(t, testModel(1))
|
||||
lead := &switchLeader{}
|
||||
r.SetLeadership(lead)
|
||||
addNode(inv, c, gpuNode("1", 8))
|
||||
|
||||
// A standby for longer than the window: its own start is long past.
|
||||
r.Reconcile(t.Context())
|
||||
c.advance(2 * DefaultConfig().SettleFor)
|
||||
touchAt(inv, "1", c.now())
|
||||
r.Reconcile(t.Context())
|
||||
|
||||
// It takes over. The agents have not told it what runs yet; placing now
|
||||
// would start a second copy of what they are about to report.
|
||||
lead.on = true
|
||||
r.Reconcile(t.Context())
|
||||
if n := len(disp.startedShards()); n != 0 {
|
||||
t.Fatalf("placed %d shard(s) the moment it took over", n)
|
||||
}
|
||||
// Once the fleet has had its window, it places what is still missing.
|
||||
c.advance(DefaultConfig().SettleFor)
|
||||
touchAt(inv, "1", c.now())
|
||||
r.Reconcile(t.Context())
|
||||
if len(disp.startedShards()) == 0 {
|
||||
t.Fatal("nothing placed after the window")
|
||||
}
|
||||
}
|
||||
@@ -270,6 +270,9 @@ type Reconciler struct {
|
||||
restartQueue map[string][]string
|
||||
// restartInPlace are the replicas asked to restart by stopping first.
|
||||
restartInPlace map[string]bool
|
||||
// restartPace is, per model, how many of its replicas may be restarting
|
||||
// at once: one unless an operator said otherwise.
|
||||
restartPace map[string]RestartPace
|
||||
// restartRefused says, by replica, why a restart was given up rather
|
||||
// than leave the model without it.
|
||||
restartRefused map[string]string
|
||||
@@ -304,6 +307,12 @@ type Reconciler struct {
|
||||
startedAt time.Time
|
||||
lastNodeCount int
|
||||
steadySince time.Time
|
||||
// stopped is the operators having stopped the service: every model is
|
||||
// wanted at no replicas until they start it again.
|
||||
stopped atomic.Bool
|
||||
// stoodBy is whether the last pass found this process a standby: one
|
||||
// that then takes over settles again, as at start.
|
||||
stoodBy bool
|
||||
}
|
||||
|
||||
// New builds a reconciler.
|
||||
@@ -323,6 +332,7 @@ func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, l
|
||||
wanted: map[string]time.Time{},
|
||||
restartQueue: map[string][]string{},
|
||||
restartInPlace: map[string]bool{},
|
||||
restartPace: map[string]RestartPace{},
|
||||
restartRefused: map[string]string{},
|
||||
parked: map[string]parkRecord{},
|
||||
extra: map[string]int{},
|
||||
@@ -408,8 +418,22 @@ func (r *Reconciler) Reconcile(ctx context.Context) {
|
||||
// Unset means leader, so a single control plane and every test behave
|
||||
// exactly as they did before leadership existed.
|
||||
if r.leader != nil && !r.leader.IsLeader() {
|
||||
r.stoodBy = true
|
||||
return
|
||||
}
|
||||
// Taking over from another control plane is a start as far as the fleet
|
||||
// is concerned: a standby refuses agents, so it knows none of what runs,
|
||||
// and placing before they have reconnected and said starts a second
|
||||
// copy of every replica — then drains the healthy one it finds next.
|
||||
// Measured on the cluster: a failover replaced the guardrails' judge and
|
||||
// a model's replica within twenty seconds, for nothing.
|
||||
// From the moment it leads, not from when the process started: a
|
||||
// standby a minute old had its window run out while it was refusing the
|
||||
// very agents the window waits for.
|
||||
if r.stoodBy {
|
||||
r.stoodBy = false
|
||||
r.resettle()
|
||||
}
|
||||
ctx, p := beginPass(ctx)
|
||||
defer p.end()
|
||||
r.expireNodes(ctx)
|
||||
@@ -463,6 +487,25 @@ func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// SetStopped stops every model, or lets each run as its catalog entry
|
||||
// asks again, from the next pass.
|
||||
func (r *Reconciler) SetStopped(stopped bool) {
|
||||
if r.stopped.Swap(stopped) != stopped {
|
||||
r.Notify()
|
||||
}
|
||||
}
|
||||
|
||||
// Stopped is whether the operators have stopped the service.
|
||||
func (r *Reconciler) Stopped() bool { return r.stopped.Load() }
|
||||
|
||||
// resettle opens the settling window again from now, for a control plane
|
||||
// that has just become the leader.
|
||||
func (r *Reconciler) resettle() {
|
||||
// Like the rest of the window, touched only by Reconcile's own pass.
|
||||
r.startedAt, r.lastNodeCount, r.steadySince = r.now(), -1, r.now()
|
||||
r.log.Info("leading now: placement waits for the fleet to report what runs", "for up to", r.cfg.SettleFor)
|
||||
}
|
||||
|
||||
// fleetHasSettled reports whether placement may run yet.
|
||||
//
|
||||
// Two ways out, because either alone is wrong. Waiting for the window to expire
|
||||
@@ -775,19 +818,37 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
||||
drainingIDs = append(drainingIDs, rep.ID)
|
||||
}
|
||||
}
|
||||
// The next restart an operator queued, once the last one is done.
|
||||
// The next restarts an operator queued, as many at once as the pace
|
||||
// allows: one by default, each begun once the last is done.
|
||||
restarting := ""
|
||||
if len(retiringIDs) == 0 {
|
||||
if id, ok := r.nextRestartLocked(m.Name); ok {
|
||||
r.replicas[id].RetiringAt = r.now()
|
||||
retiringIDs = append(retiringIDs, id)
|
||||
liveIDs = slices.DeleteFunc(liveIDs, func(x string) bool { return x == id })
|
||||
readyLive--
|
||||
restarting = id
|
||||
r.log.Info("restarting a replica: placing its replacement first",
|
||||
"replica", id, "model", m.Name)
|
||||
pace := r.restartPace[m.Name]
|
||||
inPlaceNow := 0
|
||||
for _, id := range retiringIDs {
|
||||
if r.restartInPlace[id] {
|
||||
inPlaceNow++
|
||||
}
|
||||
}
|
||||
for len(retiringIDs) < pace.width() {
|
||||
id, ok := r.nextRestartLocked(m.Name)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
// Up to MaxUnavailable of them stop first; the rest are surged.
|
||||
if inPlaceNow < pace.MaxUnavailable {
|
||||
r.restartInPlace[id] = true
|
||||
inPlaceNow++
|
||||
}
|
||||
r.replicas[id].RetiringAt = r.now()
|
||||
retiringIDs = append(retiringIDs, id)
|
||||
liveIDs = slices.DeleteFunc(liveIDs, func(x string) bool { return x == id })
|
||||
readyLive--
|
||||
restarting = id
|
||||
r.log.Info("restarting a replica", "replica", id, "model", m.Name,
|
||||
"in_place", r.restartInPlace[id], "at_once", pace.width())
|
||||
}
|
||||
if _, queued := r.restartQueue[m.Name]; !queued && len(retiringIDs) == 0 {
|
||||
delete(r.restartPace, m.Name)
|
||||
}
|
||||
blockedUntil := r.nextAttempt[m.Name]
|
||||
park, isParked := r.parked[m.Name]
|
||||
if isParked && park.fp != modelFingerprint(m) {
|
||||
@@ -1246,6 +1307,22 @@ func (r *Reconciler) OnFetchProgress(replicaID, shardID, source string, done, to
|
||||
}
|
||||
}
|
||||
|
||||
// OnShardCompute records where a shard's engine said it put the model.
|
||||
func (r *Reconciler) OnShardCompute(replicaID, shardID string, c Compute) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
rep, ok := r.replicas[replicaID]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
for i := range rep.Shards {
|
||||
if rep.Shards[i].ID == shardID {
|
||||
cc := c
|
||||
rep.Shards[i].Compute = &cc
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// OnShardStatus records a status report from an agent.
|
||||
func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
|
||||
r.markShard(replicaID, shardID, st, errMsg)
|
||||
|
||||
@@ -2063,3 +2063,140 @@ func TestLoadAddsReplicasUpToMaxAndQuietTakesThemBack(t *testing.T) {
|
||||
t.Errorf("%d replicas after a long quiet spell; min is 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplicasRestartSeveralAtOnceWhenAsked(t *testing.T) {
|
||||
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8), gpuNode("4", 8))
|
||||
for i := 0; i < 3; i++ {
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
}
|
||||
var ids []string
|
||||
for _, rep := range h.r.ReadyReplicas() {
|
||||
ids = append(ids, rep.ID)
|
||||
}
|
||||
if err := h.r.RestartPaced(RestartPace{MaxSurge: 2}, false, ids...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.r.Reconcile(t.Context())
|
||||
replacing, queued := 0, 0
|
||||
for _, id := range ids {
|
||||
switch h.r.RestartState(id) {
|
||||
case "replacing":
|
||||
replacing++
|
||||
case "queued":
|
||||
queued++
|
||||
}
|
||||
}
|
||||
if replacing != 2 || queued != 1 {
|
||||
t.Errorf("max surge 2: %d replacing, %d queued", replacing, queued)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxUnavailableStopsThatManyFirst(t *testing.T) {
|
||||
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
||||
for i := 0; i < 3; i++ {
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
}
|
||||
var ids []string
|
||||
for _, rep := range h.r.ReadyReplicas() {
|
||||
ids = append(ids, rep.ID)
|
||||
}
|
||||
if err := h.r.RestartPaced(RestartPace{MaxSurge: 1, MaxUnavailable: 1}, false, ids...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.r.Reconcile(t.Context())
|
||||
stopped, beside := 0, 0
|
||||
for _, id := range ids {
|
||||
rep, ok := replicaByID(h, id)
|
||||
switch {
|
||||
case !ok || rep.State != Ready:
|
||||
stopped++
|
||||
case h.r.RestartState(id) == "replacing":
|
||||
beside++
|
||||
}
|
||||
}
|
||||
if stopped != 1 || beside != 1 {
|
||||
t.Errorf("one stopped first and one replaced beside itself: %d stopped, %d beside", stopped, beside)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdoptedShardKeepsWhatItComputesOn(t *testing.T) {
|
||||
// A control plane that took over, or restarted, learns where a replica
|
||||
// computes from what its agent kept, not from a READY it never saw.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
a := adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})
|
||||
a.Compute = &Compute{Backend: "CUDA", Devices: []string{"CUDA0: H100"}, Offloaded: 81, Total: 81}
|
||||
h.r.Adopt("1", []AdoptedShard{a})
|
||||
reps := h.r.Replicas()
|
||||
if len(reps) != 1 || reps[0].Shards[0].Compute == nil || reps[0].Shards[0].Compute.Offloaded != 81 {
|
||||
t.Fatalf("%+v", reps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoppingTheServiceTakesEveryModelDownAndStartingBringsItBack(t *testing.T) {
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
if len(h.r.ReadyReplicas()) != 1 {
|
||||
t.Fatal("nothing ready to stop")
|
||||
}
|
||||
h.r.SetStopped(true)
|
||||
h.r.Reconcile(t.Context())
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.State == Ready {
|
||||
t.Fatalf("stopped, a replica still serves: %+v", rep)
|
||||
}
|
||||
}
|
||||
h.r.SetStopped(false)
|
||||
for range 3 {
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
}
|
||||
if len(h.r.ReadyReplicas()) != 1 {
|
||||
t.Errorf("started again, %d ready", len(h.r.ReadyReplicas()))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRoomAnInPlaceRestartFreesIsItsReplacements(t *testing.T) {
|
||||
// Seen on the reference cluster: a model restarted in place on the only
|
||||
// GPU, a judge waiting for that card, and the judge placed on it in the
|
||||
// same pass, before the replacement — the model was gone until someone
|
||||
// stopped the judge by hand.
|
||||
for _, waitingFirst := range []bool{false, true} {
|
||||
m := testModel(1)
|
||||
waiting := testModel(1)
|
||||
waiting.Name, waiting.Source = "judge", "/models/judge"
|
||||
models := []catalog.Model{m, waiting}
|
||||
if waitingFirst {
|
||||
models = []catalog.Model{waiting, m}
|
||||
}
|
||||
h := newHarness(t, m, gpuNode("1", 4))
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
old := h.r.ReadyReplicas()[0].ID
|
||||
h.setModels(t, models...)
|
||||
h.r.Reconcile(t.Context()) // the judge finds no room
|
||||
|
||||
if err := h.r.Restart(old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.r.Reconcile(t.Context())
|
||||
var ofModel, ofWaiting int
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.ID == old || !rep.State.Counts() {
|
||||
continue
|
||||
}
|
||||
switch rep.Model {
|
||||
case m.Name:
|
||||
ofModel++
|
||||
case waiting.Name:
|
||||
ofWaiting++
|
||||
}
|
||||
}
|
||||
if ofModel != 1 || ofWaiting != 0 {
|
||||
t.Errorf("waiting first=%v: the freed GPUs went to %d replacement(s) and %d of the waiting model",
|
||||
waitingFirst, ofModel, ofWaiting)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -72,6 +72,20 @@ type Shard struct {
|
||||
// shard reaches any other state, so a finished download does not sit in
|
||||
// the listing looking like one still running.
|
||||
Fetch *FetchProgress
|
||||
|
||||
// Compute is where the engine said it put the model as it became ready:
|
||||
// nil when it did not say.
|
||||
Compute *Compute
|
||||
}
|
||||
|
||||
// Compute is where an engine runs its model, by its own account.
|
||||
type Compute struct {
|
||||
// Backend is CUDA, Vulkan, Metal, ROCm, GPU or CPU.
|
||||
Backend string
|
||||
// Devices it named: "CUDA0: Quadro RTX 6000 (7.5)".
|
||||
Devices []string
|
||||
// Offloaded of Total layers went to the devices; 0 and 0 when unsaid.
|
||||
Offloaded, Total int
|
||||
}
|
||||
|
||||
// FetchProgress is how far a shard's weights have got.
|
||||
|
||||
@@ -73,11 +73,45 @@ func (r *Reconciler) retire(ctx context.Context, m catalog.Model, ids []string,
|
||||
}
|
||||
}
|
||||
|
||||
// stopForRestart drains a replica and stops it at once, for its replacement
|
||||
// to take its place.
|
||||
// stopForRestart drains a replica and stops it at once, and places its
|
||||
// replacement in what it freed.
|
||||
//
|
||||
// At once, not on the next pass: the pass goes on to the other models, and
|
||||
// one waiting for room took the freed cards first. Seen on the reference
|
||||
// cluster with a judge waiting for the only GPU: the model restarted there
|
||||
// lost its card to the judge.
|
||||
func (r *Reconciler) stopForRestart(ctx context.Context, id, why string) {
|
||||
r.mu.Lock()
|
||||
rep, ok := r.replicas[id]
|
||||
model := ""
|
||||
if ok {
|
||||
model = rep.Model
|
||||
}
|
||||
r.mu.Unlock()
|
||||
r.drainForRestart(ctx, id, why)
|
||||
r.stopReplica(ctx, id, why)
|
||||
m, found := r.cat.Load().Resolve(model)
|
||||
if !ok || !found || m.Name != model {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
var avoid []string
|
||||
live := 0
|
||||
for _, o := range r.replicas {
|
||||
// As reconcileModel counts: one being retired is not one it has.
|
||||
if o.Model == model && o.State.Counts() && o.RetiringAt.IsZero() {
|
||||
live++
|
||||
avoid = append(avoid, o.NodeIDs()...)
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
if live >= r.desiredFor(m) {
|
||||
return
|
||||
}
|
||||
if outcome := r.scaleUp(ctx, m, avoid); outcome != placed {
|
||||
r.log.Warn("the replacement of a replica restarted in place was not placed in its room",
|
||||
"replica", id, "model", model, "outcome", outcome.String())
|
||||
}
|
||||
}
|
||||
|
||||
// refuseRestart gives up a replica's restart, leaving it serving, and keeps
|
||||
@@ -127,6 +161,42 @@ func (r *Reconciler) nextRestartLocked(model string) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
// RestartPace is how many of a model's replicas restart at once.
|
||||
type RestartPace struct {
|
||||
// MaxSurge are replaced beside themselves at once: their replacements
|
||||
// placed first, then they stop. Zero is one.
|
||||
MaxSurge int
|
||||
// MaxUnavailable are stopped first at once, their replacements starting
|
||||
// in their place: an outage for each while it loads, needing no room.
|
||||
MaxUnavailable int
|
||||
}
|
||||
|
||||
// width is how many may be restarting at once.
|
||||
func (p RestartPace) width() int {
|
||||
surge := p.MaxSurge
|
||||
if surge <= 0 && p.MaxUnavailable <= 0 {
|
||||
surge = 1
|
||||
}
|
||||
return surge + max(p.MaxUnavailable, 0)
|
||||
}
|
||||
|
||||
// RestartPaced queues replicas like Restart (or RestartInPlace), as many of
|
||||
// each model's at once as pace says.
|
||||
func (r *Reconciler) RestartPaced(pace RestartPace, inPlace bool, ids ...string) error {
|
||||
if err := r.restart(inPlace, ids); err != nil {
|
||||
return err
|
||||
}
|
||||
r.mu.Lock()
|
||||
for _, id := range ids {
|
||||
if rep, ok := r.replicas[id]; ok {
|
||||
r.restartPace[rep.Model] = pace
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
r.Notify()
|
||||
return nil
|
||||
}
|
||||
|
||||
// ErrNoSuchReplica means no ready replica has that id.
|
||||
var ErrNoSuchReplica = errors.New("no ready replica with that id")
|
||||
|
||||
|
||||
@@ -121,15 +121,17 @@ var Reference = []Block{
|
||||
{Key: "enabled", Type: "bool", Doc: "Enabled keeps llmgw's internal ports — engines, node exporter, the\nagents' stream, the mirror, the store — to the cluster's networks on\nevery machine. False removes llmgw's nftables table everywhere, for a\nsite whose own firewall does that job; the rules are kept, and true\nputs them back. Changed in the cluster's settings, it is in force on\nevery node and control plane within seconds.", Nested: ""},
|
||||
}},
|
||||
{Package: "catalog", Name: "GuardModel", Title: "GuardModel", Doc: "", Fields: []Field{
|
||||
{Key: "format", Type: "string", Doc: "Format is llamaguard3 (Meta's Llama Guard 3: \"safe\", or \"unsafe\" and\nthe categories), shieldgemma (Google's ShieldGemma: Yes or No), or\njson ({\"safe\": false, \"categories\": [\"...\"]}).", Nested: ""},
|
||||
{Key: "format", Type: "string", Doc: "Format is qwen3guard (Qwen3Guard-Gen: \"Safety: Safe\", \"Unsafe\" or\n\"Controversial\", and the categories), llamaguard3 (Meta's Llama Guard\n3: \"safe\", or \"unsafe\" and the categories), shieldgemma (Google's\nShieldGemma: Yes or No), or json ({\"safe\": false, \"categories\":\n[\"...\"]}).", Nested: ""},
|
||||
{Key: "categories", Type: "[]string", Doc: "Categories the model is asked about, for shieldgemma — each a policy\nsentence — and the names reported for llamaguard3's S1, S2, ….", Nested: ""},
|
||||
}},
|
||||
{Package: "guard", Name: "JudgePolicy", Title: "JudgePolicy", Doc: "", Fields: []Field{
|
||||
{Key: "model", Type: "string", Doc: "Model is a catalog model with role: guard.", Nested: ""},
|
||||
{Key: "models", Type: "[]string", Doc: "Models are judges asked in turn, instead of Model: the first that\nanswers gives the verdict. A judge on a GPU first and one on CPUs\nafter it is the GPU's speed while it is there, and the CPUs' when it\nis not — busy, failed, or the node down.", Nested: ""},
|
||||
{Key: "input", Type: "string", Doc: "Input and Output are what an unsafe verdict does: flag or block.\nEmpty does not ask the judge about that direction.", Nested: ""},
|
||||
{Key: "output", Type: "string", Doc: "", Nested: ""},
|
||||
{Key: "categories", Type: "[]string", Doc: "Categories, when set, are the only ones that count: Llama Guard's\ncodes (S1, S10) or the names the model reports.", Nested: ""},
|
||||
{Key: "latency_budget", Type: "time.Duration", Doc: "LatencyBudget is how long the judge is waited for before the policy's\nfail applies. Zero is 250ms.", Nested: ""},
|
||||
{Key: "controversial", Type: "string", Doc: "Controversial is what Qwen3Guard's third verdict counts as: safe\n(the default) or unsafe.", Nested: ""},
|
||||
{Key: "latency_budget", Type: "time.Duration", Doc: "LatencyBudget is how long each judge is waited for before the next is\nasked, or, after the last, the policy's fail applies. Zero is 250ms.", Nested: ""},
|
||||
}},
|
||||
{Package: "config", Name: "KVIndex", Title: "KVIndex", Doc: "", Fields: []Field{
|
||||
{Key: "tokenize_timeout", Type: "time.Duration", Doc: "TokenizeTimeout bounds the call that turns a request into token ids,\nmade to one of the model's replicas before the request is placed. Past\nit the request is routed as tier 1 would. Zero takes 250ms.", Nested: ""},
|
||||
@@ -261,6 +263,7 @@ var Reference = []Block{
|
||||
{Package: "catalog", Name: "Profile", Title: "Profile", Doc: "", Fields: []Field{
|
||||
{Key: "name", Type: "string", Doc: "", Nested: ""},
|
||||
{Key: "engine", Type: "string", Doc: "", Nested: ""},
|
||||
{Key: "engine_version", Type: "string", Doc: "EngineVersion restricts the profile to nodes whose installed engine's\nversion starts with it: \"0.11\" takes vLLM 0.11.0 and 0.11.1, not\n0.10.2. Empty takes whatever version the node has.", Nested: ""},
|
||||
{Key: "gpu_class", Type: "string", Doc: "", Nested: ""},
|
||||
{Key: "tp", Type: "int", Doc: "", Nested: ""},
|
||||
{Key: "pp", Type: "int", Doc: "", Nested: ""},
|
||||
@@ -366,6 +369,8 @@ var Reference = []Block{
|
||||
{Key: "guard", Type: "guard.Config", Doc: "Guard is the library of guardrail policies: what requests and answers\nare checked for, and what is done. Applied to every request through\nguard.default, and to parts of the tenancy tree by attaching them\n(llmgw ctl guard attach). Empty checks nothing.", Nested: "guard.Config"},
|
||||
{Key: "secrets", Type: "Secrets", Doc: "Secrets is where secrets kept outside files are read from, and the key\nthe store's own secrets are sealed under.", Nested: "config.Secrets"},
|
||||
{Key: "notify", Type: "notify.Config", Doc: "Notify sends what needs a person — a model parked, a node lost, a\nbudget spent, new weights published — to webhooks, signed, without a\nPrometheus and Alertmanager in between. Empty sends nothing.", Nested: "notify.Config"},
|
||||
{Key: "mail", Type: "mail.Config", Doc: "Mail is where the gateway's few emails go out from: a link to set a\nforgotten password. Unset, none is sent and people ask an operator.", Nested: "mail.Config"},
|
||||
{Key: "serving", Type: "ServingSettings", Doc: "Serving stops and starts the whole cluster's service (llmgw stop, llmgw\nstart).", Nested: "config.ServingSettings"},
|
||||
{Key: "firewall", Type: "FirewallSettings", Doc: "Firewall turns llmgw's firewall on every machine on or off.", Nested: "config.FirewallSettings"},
|
||||
{Key: "firewall_networks", Type: "[]string", Doc: "FirewallNetworks are networks, besides those each machine reaches the\ncluster on, that llmgw's internal ports answer: a monitoring host on\nanother subnet, nodes behind a router. Written into every machine's\nfirewall by `server init`, `node add` and `server add`.", Nested: ""},
|
||||
{Key: "backup_recipients", Type: "[]string", Doc: "BackupRecipients encrypt every archive `llmgw server backup` writes, to\nwhoever holds one of their keys: age recipients (age1…) or SSH public\nkeys (ssh-ed25519 …, ssh-rsa …), as `age` takes them. An archive holds\nthe certificate authority's key and every secret in the store, and is\ncopied off the machine; encrypted, the copy is safe wherever it lands.\nEmpty writes archives unencrypted, readable by root only.", Nested: ""},
|
||||
@@ -379,6 +384,9 @@ var Reference = []Block{
|
||||
{Key: "federation", Type: "Federation", Doc: "Federation is how this cluster takes part in federation with other\nllmgw clusters. The peers themselves are added at runtime, with\n`llmgw ctl federation add`, and kept in the store.", Nested: "config.Federation"},
|
||||
{Key: "endpoints", Type: "[]Endpoint", Doc: "Endpoints is the phase-1 static replica list. It is only consulted when\nno model declares a profile, which is what makes a router-only smoke test\nagainst sim-engines possible without a control plane.", Nested: ""},
|
||||
}},
|
||||
{Package: "config", Name: "ServingSettings", Title: "ServingSettings", Doc: "", Fields: []Field{
|
||||
{Key: "stopped", Type: "bool", Doc: "Stopped takes every model down and answers every request for one with\n503 (system_stopped), leaving the control planes, agents and mirror\nrunning and every model's definition as it is: llmgw stop sets it,\nllmgw start clears it, and the models come back as their catalog\nentries ask. In force on every control plane within seconds.", Nested: ""},
|
||||
}},
|
||||
{Package: "engines", Name: "SimConfig", Title: "SimConfig", Doc: "", Fields: []Field{
|
||||
{Key: "bin", Type: "string", Doc: "Bin is the llmgw binary itself; the engine is one of its subcommands.\nEmpty means whatever started this agent, which is right almost always.", Nested: ""},
|
||||
}},
|
||||
@@ -484,9 +492,17 @@ var Reference = []Block{
|
||||
{Key: "sim", Type: "SimConfig", Doc: "", Nested: "engines.SimConfig"},
|
||||
}},
|
||||
{Package: "guard", Name: "Config", Title: "guard.Config", Doc: "", Fields: []Field{
|
||||
{Key: "enabled", Type: "*bool", Doc: "Enabled turns every policy on or off at once: false checks nothing,\nwhatever is defined or attached. Unset is on.", Nested: ""},
|
||||
{Key: "default", Type: "[]string", Doc: "", Nested: ""},
|
||||
{Key: "policies", Type: "map[string]Policy", Doc: "", Nested: ""},
|
||||
}},
|
||||
{Package: "mail", Name: "Config", Title: "mail.Config", Doc: "", Fields: []Field{
|
||||
{Key: "smtp", Type: "string", Doc: "SMTP is the relay, host:port: 587 with STARTTLS, 465 with TLS from the\nstart, 25 as the relay allows. Empty sends nothing.", Nested: ""},
|
||||
{Key: "from", Type: "string", Doc: "From is the sender, llmgw@example.com or \"llmgw <llmgw@example.com>\".", Nested: ""},
|
||||
{Key: "username", Type: "string", Doc: "Username and PasswordFile log in to the relay, when it wants that.\nThe password is a secret reference: a file, env:NAME or vault:path.", Nested: ""},
|
||||
{Key: "password_file", Type: "string", Doc: "", Nested: ""},
|
||||
{Key: "base_url", Type: "string", Doc: "BaseURL is the gateway's public address, which links in mail point\nat: https://llm.example.com. Never taken from a request, whose Host a\nsender chooses and could point a reset link at their own site.", Nested: ""},
|
||||
}},
|
||||
{Package: "notify", Name: "Config", Title: "notify.Config", Doc: "", Fields: []Field{
|
||||
{Key: "webhooks", Type: "[]Webhook", Doc: "", Nested: ""},
|
||||
{Key: "host", Type: "string", Doc: "Host names the gateway in messages; empty is this machine's name.", Nested: ""},
|
||||
|
||||
@@ -31,6 +31,8 @@ var Sources = []string{
|
||||
"internal/guard/judge.go",
|
||||
"internal/config/oidc.go",
|
||||
"internal/config/firewall.go",
|
||||
"internal/config/serving.go",
|
||||
"internal/mail/mail.go",
|
||||
"internal/notify/notify.go",
|
||||
"internal/secrets/secrets.go",
|
||||
"internal/obs/trace.go",
|
||||
|
||||
@@ -819,3 +819,45 @@ func sortedKeys[V any](m map[string]V) []string {
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// MultiNode is whether a finding is about the fabrics replicas spanning
|
||||
// nodes use, rather than about the node itself.
|
||||
func (f Finding) MultiNode() bool { return f.Area == "fabric" }
|
||||
|
||||
// Readiness is a node's two readinesses, kept apart: whether it can join and
|
||||
// serve replicas of its own, and whether it can also take part in replicas
|
||||
// spanning nodes over a fabric. The second never keeps a node out unless a
|
||||
// fabric was declared for it, whose problems are then the node's.
|
||||
type Readiness struct {
|
||||
// Node is "ready", or how many problems keep it out.
|
||||
Node string `json:"node"`
|
||||
// MultiNode is "ready", "not used" (no fabric adapter), or "not ready:"
|
||||
// and the first reason.
|
||||
MultiNode string `json:"multi_node"`
|
||||
}
|
||||
|
||||
// ReadinessOf sums findings up as the two readinesses.
|
||||
func ReadinessOf(fs []Finding) Readiness {
|
||||
r := Readiness{Node: "ready", MultiNode: "not used"}
|
||||
fails := 0
|
||||
for _, f := range fs {
|
||||
if f.Level == Fail {
|
||||
fails++
|
||||
}
|
||||
}
|
||||
if fails > 0 {
|
||||
r.Node = fmt.Sprintf("%d problem(s) keep it out", fails)
|
||||
}
|
||||
for _, f := range fs {
|
||||
if !f.MultiNode() {
|
||||
continue
|
||||
}
|
||||
if r.MultiNode == "not used" {
|
||||
r.MultiNode = "ready"
|
||||
}
|
||||
if (f.Level == Fail || f.Level == Warn) && !strings.HasPrefix(r.MultiNode, "not ready") {
|
||||
r.MultiNode = "not ready: " + f.Message
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
@@ -259,3 +259,20 @@ func TestAMissingFirewallIsSaidOnASetUpNode(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheNodeAndItsFabricsAreReadySeparately(t *testing.T) {
|
||||
r := ReadinessOf([]Finding{
|
||||
{Level: OK, Area: "gpu", Message: "2 GPUs"},
|
||||
{Level: Warn, Area: "fabric", Message: "mlx5_1 is not active"},
|
||||
})
|
||||
if r.Node != "ready" || r.MultiNode != "not ready: mlx5_1 is not active" {
|
||||
t.Errorf("%+v", r)
|
||||
}
|
||||
r = ReadinessOf([]Finding{{Level: Fail, Area: "gpu", Message: "no driver"}})
|
||||
if !strings.Contains(r.Node, "1 problem") || r.MultiNode != "not used" {
|
||||
t.Errorf("%+v", r)
|
||||
}
|
||||
if r := ReadinessOf([]Finding{{Level: OK, Area: "fabric", Message: "mlx5_0 active"}}); r.MultiNode != "ready" {
|
||||
t.Errorf("%+v", r)
|
||||
}
|
||||
}
|
||||
+53
-3
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -43,6 +44,9 @@ type Gate struct {
|
||||
m *gateMetrics
|
||||
// alarm is told each time a judge gives no verdict; nil tells no one.
|
||||
alarm func(model string, err error)
|
||||
// lastErr is each judge's last failure to give a verdict.
|
||||
errMu sync.Mutex
|
||||
lastErr map[string]JudgeError
|
||||
|
||||
mu sync.Mutex
|
||||
attached map[string][]string // "scope/id" → policy names
|
||||
@@ -152,6 +156,10 @@ func (g *Gate) Invalidate() { g.reload(context.Background()) }
|
||||
// from.
|
||||
func (g *Gate) Policy(s Subject) (Policy, string) {
|
||||
g.mu.Lock()
|
||||
if !g.cfg.On() {
|
||||
g.mu.Unlock()
|
||||
return Policy{}, ""
|
||||
}
|
||||
if !g.loading && g.store != nil && g.now().Sub(g.loadedAt) > attachmentsTTL {
|
||||
g.loading = true
|
||||
go g.reload(context.WithoutCancel(context.Background()))
|
||||
@@ -324,7 +332,7 @@ func (g *Gate) Input(ctx context.Context, s Subject, body []byte) (Verdict, erro
|
||||
done := func(f finding) Verdict {
|
||||
v.Action, v.Category, v.Rule = f.action, f.category, f.rule
|
||||
if f.action == Block {
|
||||
v.Message = fmt.Sprintf("this request was refused by the gateway's policy (%s)", f.category)
|
||||
v.Message = fmt.Sprintf("this request was refused by the gateway's policy (%s)", refusedFor(f))
|
||||
v.Body = nil
|
||||
}
|
||||
v.Took = g.now().Sub(start)
|
||||
@@ -402,8 +410,8 @@ func (g *Gate) Input(ctx context.Context, s Subject, body []byte) (Verdict, erro
|
||||
if jp := p.Judge; jp != nil && jp.Input != "" && user != "" {
|
||||
res, err := g.judger.ask(ctx, jp, user, "")
|
||||
if err != nil {
|
||||
g.log.Warn("the judge gave no verdict on a request", "judge", jp.Model, "err", err)
|
||||
g.judgeFailed(jp.Model, err)
|
||||
g.log.Warn("the judge gave no verdict on a request", "judge", jp.Name(), "err", err)
|
||||
g.noJudger(jp, err)
|
||||
}
|
||||
if err != nil && p.Fail == "closed" && !v.Shadow {
|
||||
v.Took = g.now().Sub(start)
|
||||
@@ -447,12 +455,43 @@ func (g *Gate) Input(ctx context.Context, s Subject, body []byte) (Verdict, erro
|
||||
// the gate serves.
|
||||
func (g *Gate) SetJudgeAlarm(f func(model string, err error)) { g.alarm = f }
|
||||
|
||||
// noJudger records, against each judge a policy asks, that there was no way
|
||||
// to ask any: a gate built without SetJudge. With one, each judge's failure
|
||||
// is recorded as it is asked.
|
||||
func (g *Gate) noJudger(jp *JudgePolicy, err error) {
|
||||
if g.judger != nil {
|
||||
return
|
||||
}
|
||||
for _, m := range jp.Judges() {
|
||||
g.judgeFailed(m, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Gate) judgeFailed(model string, err error) {
|
||||
g.errMu.Lock()
|
||||
if g.lastErr == nil {
|
||||
g.lastErr = map[string]JudgeError{}
|
||||
}
|
||||
g.lastErr[model] = JudgeError{Err: err.Error(), At: g.now()}
|
||||
g.errMu.Unlock()
|
||||
if g.alarm != nil {
|
||||
g.alarm(model, err)
|
||||
}
|
||||
}
|
||||
|
||||
// JudgeError is a judge's last failure to give a verdict.
|
||||
type JudgeError struct {
|
||||
Err string
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// JudgeErrors are each judge's last failure to give a verdict, by model.
|
||||
func (g *Gate) JudgeErrors() map[string]JudgeError {
|
||||
g.errMu.Lock()
|
||||
defer g.errMu.Unlock()
|
||||
return maps.Clone(g.lastErr)
|
||||
}
|
||||
|
||||
func judgeFinding(p Policy, action string, res judgment, err error) finding {
|
||||
if err != nil {
|
||||
// Unanswered: closed stops it, open lets it through, said.
|
||||
@@ -501,3 +540,14 @@ func toolAllowed(t Tools, name string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// refusedFor is what a refusal names: the category, and for a judge the
|
||||
// hazard it saw — "judge: S6,Specialized Advice" — so whoever is refused
|
||||
// can say why, and a judge refusing ordinary questions can be told apart.
|
||||
// Never the text.
|
||||
func refusedFor(f finding) string {
|
||||
if f.category == "judge" && f.rule != "" && f.rule != "unavailable" {
|
||||
return "judge: " + f.rule
|
||||
}
|
||||
return f.category
|
||||
}
|
||||
@@ -278,3 +278,34 @@ func TestCapabilitiesAndStructuredAnswersWhenAskedFor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheGuardrailsTurnOffAndOnAsAWhole(t *testing.T) {
|
||||
off := false
|
||||
cfg := lib(map[string]Policy{"base": {Input: Rules{PII: map[string]string{"secret": "block"}}}}, "base")
|
||||
cfg.Enabled = &off
|
||||
g := gate(t, cfg)
|
||||
secret := chat("user", "my key is sk-proj-abcdefghijklmnopqrstuvwx1234")
|
||||
if v, err := g.Input(context.Background(), Subject{}, secret); err != nil || v.Action != "" {
|
||||
t.Errorf("off, a secret was acted on: %+v %v", v, err)
|
||||
}
|
||||
on := true
|
||||
cfg.Enabled = &on
|
||||
if err := g.Replace(cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, _ := g.Input(context.Background(), Subject{}, secret); v.Action != Block {
|
||||
t.Errorf("on again, a secret went through: %+v", v)
|
||||
}
|
||||
if !(Config{}).On() {
|
||||
t.Error("unset is not on")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAJudgesRefusalNamesTheHazard(t *testing.T) {
|
||||
if got := refusedFor(finding{category: "judge", rule: "S6,Specialized Advice"}); got != "judge: S6,Specialized Advice" {
|
||||
t.Errorf("%q", got)
|
||||
}
|
||||
if got := refusedFor(finding{category: "secret", rule: "pii:secret"}); got != "secret" {
|
||||
t.Errorf("%q", got)
|
||||
}
|
||||
}
|
||||
+328
-31
@@ -10,6 +10,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard/detect"
|
||||
)
|
||||
@@ -19,6 +20,11 @@ import (
|
||||
type JudgePolicy struct {
|
||||
// Model is a catalog model with role: guard.
|
||||
Model string `koanf:"model"`
|
||||
// Models are judges asked in turn, instead of Model: the first that
|
||||
// answers gives the verdict. A judge on a GPU first and one on CPUs
|
||||
// after it is the GPU's speed while it is there, and the CPUs' when it
|
||||
// is not — busy, failed, or the node down.
|
||||
Models []string `koanf:"models"`
|
||||
// Input and Output are what an unsafe verdict does: flag or block.
|
||||
// Empty does not ask the judge about that direction.
|
||||
Input string `koanf:"input"`
|
||||
@@ -26,11 +32,31 @@ type JudgePolicy struct {
|
||||
// Categories, when set, are the only ones that count: Llama Guard's
|
||||
// codes (S1, S10) or the names the model reports.
|
||||
Categories []string `koanf:"categories"`
|
||||
// LatencyBudget is how long the judge is waited for before the policy's
|
||||
// fail applies. Zero is 250ms.
|
||||
// Controversial is what Qwen3Guard's third verdict counts as: safe
|
||||
// (the default) or unsafe.
|
||||
Controversial string `koanf:"controversial"`
|
||||
// LatencyBudget is how long each judge is waited for before the next is
|
||||
// asked, or, after the last, the policy's fail applies. Zero is 250ms.
|
||||
LatencyBudget time.Duration `koanf:"latency_budget"`
|
||||
}
|
||||
|
||||
// Judges are the judge models asked, in the order they are asked.
|
||||
func (j *JudgePolicy) Judges() []string {
|
||||
if j == nil {
|
||||
return nil
|
||||
}
|
||||
if len(j.Models) > 0 {
|
||||
return j.Models
|
||||
}
|
||||
if j.Model != "" {
|
||||
return []string{j.Model}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Name is the judges as one name, for logs and errors: "a, else b".
|
||||
func (j *JudgePolicy) Name() string { return strings.Join(j.Judges(), ", else ") }
|
||||
|
||||
// DefaultJudgeBudget is how long a judge is waited for when a policy does
|
||||
// not say.
|
||||
const DefaultJudgeBudget = 250 * time.Millisecond
|
||||
@@ -43,8 +69,26 @@ func (j *JudgePolicy) budget() time.Duration {
|
||||
}
|
||||
|
||||
func (j *JudgePolicy) validate() error {
|
||||
if j.Model == "" {
|
||||
return errors.New("judge.model is required")
|
||||
switch {
|
||||
case j.Model == "" && len(j.Models) == 0:
|
||||
return errors.New("judge.model (or judge.models, judges asked in turn) is required")
|
||||
case j.Model != "" && len(j.Models) > 0:
|
||||
return errors.New("judge: give model or models, not both")
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, m := range j.Models {
|
||||
if strings.TrimSpace(m) == "" {
|
||||
return errors.New("judge.models has an empty name")
|
||||
}
|
||||
if seen[m] {
|
||||
return fmt.Errorf("judge.models names %s twice", m)
|
||||
}
|
||||
seen[m] = true
|
||||
}
|
||||
switch j.Controversial {
|
||||
case "", "safe", "unsafe":
|
||||
default:
|
||||
return fmt.Errorf("judge.controversial is %q: safe or unsafe", j.Controversial)
|
||||
}
|
||||
for k, a := range map[string]string{"input": j.Input, "output": j.Output} {
|
||||
switch a {
|
||||
@@ -59,8 +103,8 @@ func (j *JudgePolicy) validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// mergeJudge keeps the first judge model asked for, the strictest actions,
|
||||
// every category and the shortest wait.
|
||||
// mergeJudge keeps the first judges asked for, the strictest actions, every
|
||||
// category and the shortest wait.
|
||||
func mergeJudge(a, b *JudgePolicy) *JudgePolicy {
|
||||
if b == nil {
|
||||
return a
|
||||
@@ -68,11 +112,14 @@ func mergeJudge(a, b *JudgePolicy) *JudgePolicy {
|
||||
if a == nil {
|
||||
c := *b
|
||||
c.Categories = slices.Clone(b.Categories)
|
||||
c.Models = slices.Clone(b.Models)
|
||||
return &c
|
||||
}
|
||||
out := *a
|
||||
out.Models = slices.Clone(a.Models)
|
||||
out.Input = stricterOf(a.Input, b.Input, []string{"", "flag", "block"})
|
||||
out.Output = stricterOf(a.Output, b.Output, []string{"", "flag", "block"})
|
||||
out.Controversial = stricterOf(a.Controversial, b.Controversial, []string{"", "safe", "unsafe"})
|
||||
out.Categories = union(slices.Clone(a.Categories), b.Categories)
|
||||
if b.LatencyBudget > 0 && (a.LatencyBudget == 0 || b.LatencyBudget < a.LatencyBudget) {
|
||||
out.LatencyBudget = b.LatencyBudget
|
||||
@@ -91,8 +138,13 @@ type ModelFormat func(model string) (format string, categories []string, ok bool
|
||||
|
||||
// judgment is what a judge said.
|
||||
type judgment struct {
|
||||
unsafe bool
|
||||
categories []string
|
||||
unsafe bool
|
||||
// controversial is Qwen3Guard's middle verdict, which the policy says
|
||||
// how to count.
|
||||
controversial bool
|
||||
categories []string
|
||||
// by is the judge that said it.
|
||||
by string
|
||||
}
|
||||
|
||||
// judge asks a judge model about text, with a cache of recent answers.
|
||||
@@ -100,6 +152,9 @@ type judge struct {
|
||||
call Caller
|
||||
format ModelFormat
|
||||
m *gateMetrics
|
||||
// failed is told of each judge that gave no verdict, whether or not
|
||||
// another after it did.
|
||||
failed func(model string, err error)
|
||||
|
||||
mu sync.Mutex
|
||||
cache map[[32]byte]cached
|
||||
@@ -117,7 +172,7 @@ const (
|
||||
|
||||
// SetJudge lets policies with a judge: section ask a model on the cluster.
|
||||
func (g *Gate) SetJudge(call Caller, format ModelFormat) {
|
||||
g.judger = &judge{call: call, format: format, m: g.m, cache: map[[32]byte]cached{}}
|
||||
g.judger = &judge{call: call, format: format, m: g.m, failed: g.judgeFailed, cache: map[[32]byte]cached{}}
|
||||
}
|
||||
|
||||
// errNoJudge is a policy naming a judge the gateway cannot ask, and
|
||||
@@ -127,36 +182,57 @@ var (
|
||||
errJudgeFormat = errors.New("the judge's answer could not be read")
|
||||
)
|
||||
|
||||
// ask has the judge read conversation turns — the user's, and the
|
||||
// assistant's answer when judging output.
|
||||
func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (res judgment, err error) {
|
||||
// ask has the judges read conversation turns — the user's, and the
|
||||
// assistant's answer when judging output — each in turn until one answers.
|
||||
// The error is the last judge's, when none did.
|
||||
func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (judgment, error) {
|
||||
if j == nil {
|
||||
return judgment{}, errNoJudge
|
||||
}
|
||||
start := time.Now()
|
||||
cachedHit := false
|
||||
defer func() {
|
||||
if !cachedHit && j.m != nil {
|
||||
j.m.judged(jp.Model, time.Since(start), err)
|
||||
}
|
||||
}()
|
||||
format, names, ok := j.format(jp.Model)
|
||||
if !ok {
|
||||
return judgment{}, fmt.Errorf("%w: %q is not a model with role: guard", errNoJudge, jp.Model)
|
||||
}
|
||||
key := sha256.Sum256([]byte(jp.Model + "\x00" + detect.Normalize(user) + "\x00" + detect.Normalize(answer)))
|
||||
models := jp.Judges()
|
||||
key := sha256.Sum256([]byte(strings.Join(models, "\x01") + "\x00" + detect.Normalize(user) + "\x00" + detect.Normalize(answer)))
|
||||
j.mu.Lock()
|
||||
if c, ok := j.cache[key]; ok && time.Since(c.at) < judgeCacheTTL {
|
||||
j.mu.Unlock()
|
||||
cachedHit = true
|
||||
return c.j, nil
|
||||
}
|
||||
j.mu.Unlock()
|
||||
|
||||
err := errNoJudge
|
||||
for _, model := range models {
|
||||
var res judgment
|
||||
res, err = j.askOne(ctx, jp, model, user, answer)
|
||||
if err == nil {
|
||||
j.remember(key, res)
|
||||
return res, nil
|
||||
}
|
||||
if j.failed != nil {
|
||||
j.failed(model, err)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
// The request itself is gone: no judge after this one is wanted.
|
||||
break
|
||||
}
|
||||
}
|
||||
return judgment{}, err
|
||||
}
|
||||
|
||||
// askOne asks one judge, waiting the policy's budget for it.
|
||||
func (j *judge) askOne(ctx context.Context, jp *JudgePolicy, model, user, answer string) (res judgment, err error) {
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
if j.m != nil {
|
||||
j.m.judged(model, time.Since(start), err)
|
||||
}
|
||||
}()
|
||||
format, names, ok := j.format(model)
|
||||
if !ok {
|
||||
return judgment{}, fmt.Errorf("%w: %q is not a model with role: guard", errNoJudge, model)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, jp.budget())
|
||||
defer cancel()
|
||||
path, body := judgeRequest(jp.Model, format, names, user, answer)
|
||||
out, err := j.call(ctx, jp.Model, path, body)
|
||||
path, body := judgeRequest(model, format, names, user, answer)
|
||||
out, err := j.call(ctx, model, path, body)
|
||||
if err != nil {
|
||||
return judgment{}, err
|
||||
}
|
||||
@@ -164,6 +240,11 @@ func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (
|
||||
if err != nil {
|
||||
return judgment{}, err
|
||||
}
|
||||
res.by = model
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (j *judge) remember(key [32]byte, res judgment) {
|
||||
j.mu.Lock()
|
||||
if len(j.cache) >= judgeCacheSize {
|
||||
for k, c := range j.cache {
|
||||
@@ -174,12 +255,11 @@ func (j *judge) ask(ctx context.Context, jp *JudgePolicy, user, answer string) (
|
||||
}
|
||||
j.cache[key] = cached{j: res, at: time.Now()}
|
||||
j.mu.Unlock()
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// counts reports whether a judgment is one the policy acts on.
|
||||
func (jp *JudgePolicy) counts(res judgment) (bool, string) {
|
||||
if !res.unsafe {
|
||||
if !res.unsafe && !(res.controversial && jp.Controversial == "unsafe") {
|
||||
return false, ""
|
||||
}
|
||||
if len(jp.Categories) == 0 {
|
||||
@@ -228,13 +308,116 @@ func llamaGuard3Answer(categories []string, user, answer string) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// qwen3GuardCategories are Qwen3Guard's own, as it names them in a verdict.
|
||||
// Jailbreak is judged in a user's message only.
|
||||
var qwen3GuardCategories = []string{"Violent", "Non-violent Illegal Acts", "Sexual Content or Sexual Acts", "PII",
|
||||
"Suicide & Self-Harm", "Unethical Acts", "Politically Sensitive Topics", "Copyright Violation", "Jailbreak"}
|
||||
|
||||
// qwen3Guard is Qwen3Guard-Gen's prompt, as its chat template writes it,
|
||||
// for a user's message or, with answer, the assistant's reply to it. Written
|
||||
// out for the reason llamaGuard3Answer is, and for both directions so that a
|
||||
// judge served without --jinja reads the same words.
|
||||
func qwen3Guard(user, answer string) string {
|
||||
who, task, policy, cats, refusal, last := "USER's query", "", "", qwen3GuardCategories, "", ""
|
||||
if answer == "" {
|
||||
task = "1) Evaluate whether THE LAST USER's query belongs to safe or unsafe content based on our safety policy.\n" +
|
||||
"2) Identify the category of unsafe content found.\n"
|
||||
policy = "**Safety: Safe**: The USER's query does not contain any prohibited content or content that could lead to prohibited content.\n" +
|
||||
"**Safety: Unsafe**: The USER's query contains prohibited content.\n" +
|
||||
"**Safety: Controversial**: The USER's query does not explicitly contain prohibited content, but its intent, context, " +
|
||||
"or potential responses could be misinterpreted, misused, or exploited to generate unsafe content under certain conditions.\n"
|
||||
} else {
|
||||
who = "ASSISTANT's response"
|
||||
cats = cats[:len(cats)-1]
|
||||
task = "1) Evaluate whether THE LAST ASSISTANT's response belongs to safe or unsafe content based on our safety policy.\n" +
|
||||
"2) Identify the category of unsafe content found.\n" +
|
||||
"3) Determine if THE LAST ASSISTANT's response is a refusal to the USER's query.\n"
|
||||
policy = "**Safety: Safe**: The ASSISTANT's response does not contain any prohibited content or content that could lead to prohibited content.\n" +
|
||||
"**Safety: Unsafe**: The ASSISTANT's response contains prohibited content.\n" +
|
||||
"**Safety: Controversial**: The ASSISTANT's response is not explicitly harmful, but could become risky or offensive depending on context or audience.\n"
|
||||
refusal = "<BEGIN REFUSAL POLICY>\n" +
|
||||
"**Refusal: Yes**: The ASSISTANT refused to answer or did not comply with the USER's request.\n" +
|
||||
"**Refusal: No**: The ASSISTANT directly answered or complied with the USER's request.\n" +
|
||||
"<END REFUSAL POLICY>\n\n"
|
||||
last = "\n - The third line must be one of: 'Refusal: Yes', 'Refusal: No'."
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("<|im_start|>user\n# Task:\n")
|
||||
b.WriteString(task)
|
||||
b.WriteString("\n<BEGIN SAFETY POLICY>\n")
|
||||
b.WriteString(policy)
|
||||
b.WriteString("<END SAFETY POLICY>\n\n<BEGIN UNSAFE CONTENT CATEGORIES>\n")
|
||||
for _, c := range cats {
|
||||
b.WriteString(c)
|
||||
b.WriteString(".\n")
|
||||
}
|
||||
b.WriteString("<END UNSAFE CONTENT CATEGORIES>\n\n")
|
||||
b.WriteString(refusal)
|
||||
b.WriteString("<BEGIN CONVERSATION>\n\nUSER: ")
|
||||
b.WriteString(user)
|
||||
if answer != "" {
|
||||
b.WriteString("\n\nASSISTANT: ")
|
||||
b.WriteString(answer)
|
||||
}
|
||||
b.WriteString("\n\n<END CONVERSATION>\n\nProvide your safety assessment for ONLY THE LAST **")
|
||||
b.WriteString(who)
|
||||
b.WriteString("** in the above conversation:\n" +
|
||||
" - The first line must be one of: 'Safety: Safe', 'Safety: Unsafe', 'Safety: Controversial'.\n" +
|
||||
" - The second line should start with 'Categories:' followed by a list of any unsafe content categories, " +
|
||||
"separated by commas. If the content is safe, use 'Categories: None'.")
|
||||
b.WriteString(last)
|
||||
b.WriteString("<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// What of a text a judge reads: its start and its end, so a long one still
|
||||
// fits the judge's context. A slot of 4096 tokens, the judges' own, holds
|
||||
// the prompt and about 6000 characters of code or prose; a request an editor
|
||||
// sent with a whole file pasted into it overflowed it, the judge answered
|
||||
// nothing, and under fail: closed the request was refused.
|
||||
const (
|
||||
judgeHead = 1500
|
||||
judgeTail = 4500
|
||||
judgeAnswerHead = 1500
|
||||
judgeAnswerTail = 3000
|
||||
judgeAskedHead = 500
|
||||
judgeAskedTail = 1000
|
||||
)
|
||||
|
||||
// ends is s, or its first head and last tail bytes when it is longer, cut
|
||||
// on rune boundaries.
|
||||
func ends(s string, head, tail int) string {
|
||||
if len(s) <= head+tail {
|
||||
return s
|
||||
}
|
||||
h := head
|
||||
for h > 0 && !utf8.RuneStart(s[h]) {
|
||||
h--
|
||||
}
|
||||
t := len(s) - tail
|
||||
for t < len(s) && !utf8.RuneStart(s[t]) {
|
||||
t++
|
||||
}
|
||||
return s[:h] + "\n[…]\n" + s[t:]
|
||||
}
|
||||
|
||||
// judgeRequest is the request a judge model is sent, and where.
|
||||
func judgeRequest(model, format string, categories []string, user, answer string) (string, []byte) {
|
||||
if answer == "" {
|
||||
user = ends(user, judgeHead, judgeTail)
|
||||
} else {
|
||||
user, answer = ends(user, judgeAskedHead, judgeAskedTail), ends(answer, judgeAnswerHead, judgeAnswerTail)
|
||||
}
|
||||
if format == "llamaguard3" && answer != "" {
|
||||
b, _ := json.Marshal(map[string]any{"model": model, "prompt": llamaGuard3Answer(categories, user, answer),
|
||||
"max_tokens": 16, "temperature": 0, "stream": false})
|
||||
return "/v1/completions", b
|
||||
}
|
||||
if format == "qwen3guard" {
|
||||
b, _ := json.Marshal(map[string]any{"model": model, "prompt": qwen3Guard(user, answer),
|
||||
"max_tokens": 32, "temperature": 0, "stream": false})
|
||||
return "/v1/completions", b
|
||||
}
|
||||
var msgs []map[string]string
|
||||
switch format {
|
||||
case "llamaguard3":
|
||||
@@ -294,6 +477,10 @@ func parseJudgment(format string, names []string, body []byte, answer string) (j
|
||||
}
|
||||
switch format {
|
||||
case "llamaguard3":
|
||||
if len(names) == 0 {
|
||||
// Its own: S1 is Violent Crimes, S9 Indiscriminate Weapons.
|
||||
names = llamaGuard3Categories
|
||||
}
|
||||
lines := strings.Split(text, "\n")
|
||||
switch strings.ToLower(strings.TrimSpace(lines[0])) {
|
||||
case "safe":
|
||||
@@ -316,6 +503,38 @@ func parseJudgment(format string, names []string, body []byte, answer string) (j
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
case "qwen3guard":
|
||||
// "Safety: Unsafe\nCategories: Violent, Jailbreak", and for an
|
||||
// answer a third line, "Refusal: No", that says nothing of safety.
|
||||
var res judgment
|
||||
verdict := ""
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
k, v, ok := strings.Cut(line, ":")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
v = strings.TrimSpace(v)
|
||||
switch strings.ToLower(strings.TrimSpace(k)) {
|
||||
case "safety":
|
||||
verdict = strings.ToLower(v)
|
||||
case "categories":
|
||||
for _, c := range strings.Split(v, ",") {
|
||||
if c = strings.TrimSpace(c); c != "" && !strings.EqualFold(c, "none") {
|
||||
res.categories = append(res.categories, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
switch verdict {
|
||||
case "safe":
|
||||
return judgment{}, nil
|
||||
case "unsafe":
|
||||
res.unsafe = true
|
||||
return res, nil
|
||||
case "controversial":
|
||||
res.controversial = true
|
||||
return res, nil
|
||||
}
|
||||
case "shieldgemma":
|
||||
l := strings.ToLower(text)
|
||||
switch {
|
||||
@@ -362,10 +581,88 @@ func Moderate(ctx context.Context, call Caller, model, format string, names []st
|
||||
}
|
||||
|
||||
// DefaultCategories are the names a judge format reports its categories by
|
||||
// when the catalog gives none: Llama Guard 3's S1 to S13.
|
||||
// when the catalog gives none: Llama Guard 3's S1 to S13, Qwen3Guard's own.
|
||||
func DefaultCategories(format string) []string {
|
||||
if format == "llamaguard3" {
|
||||
switch format {
|
||||
case "llamaguard3":
|
||||
return slices.Clone(llamaGuard3Categories)
|
||||
case "qwen3guard":
|
||||
return slices.Clone(qwen3GuardCategories)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Formats are the ways a judge's answers are read.
|
||||
var Formats = []string{"llamaguard3", "qwen3guard", "shieldgemma", "json"}
|
||||
|
||||
// KnownFormat is whether a judge's answers can be read in that format.
|
||||
func KnownFormat(f string) bool { return slices.Contains(Formats, f) }
|
||||
|
||||
// FormatOfWeights is the format of the judge whose weights a source names —
|
||||
// Llama Guard, Qwen3Guard, ShieldGemma — and false for weights that are no
|
||||
// judge's.
|
||||
func FormatOfWeights(source string) (string, bool) {
|
||||
s := strings.ToLower(source)
|
||||
switch {
|
||||
case strings.Contains(s, "llama-guard"):
|
||||
return "llamaguard3", true
|
||||
case strings.Contains(s, "qwen3guard"):
|
||||
return "qwen3guard", true
|
||||
case strings.Contains(s, "shieldgemma"):
|
||||
return "shieldgemma", true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// JudgeAnswer is what a judge said of a text: its words as it gave them,
|
||||
// and as the gateway reads them.
|
||||
type JudgeAnswer struct {
|
||||
Model string
|
||||
Raw string
|
||||
Unsafe bool
|
||||
// Controversial is Qwen3Guard's middle verdict; a policy says whether
|
||||
// it counts.
|
||||
Controversial bool
|
||||
Categories []string
|
||||
Took time.Duration
|
||||
}
|
||||
|
||||
// AskJudge has a judge read a text, past the cache, and says what it
|
||||
// answered: for an operator finding out why ordinary requests are refused.
|
||||
// With answer, the judge reads it as the assistant's reply to text.
|
||||
func (g *Gate) AskJudge(ctx context.Context, model, text, answer string) (JudgeAnswer, error) {
|
||||
out := JudgeAnswer{Model: model}
|
||||
if g.judger == nil {
|
||||
return out, errNoJudge
|
||||
}
|
||||
format, names, ok := g.judger.format(model)
|
||||
if !ok {
|
||||
return out, fmt.Errorf("%w: %q is not a model with role: guard", errNoJudge, model)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
path, body := judgeRequest(model, format, names, text, answer)
|
||||
reply, err := g.judger.call(ctx, model, path, body)
|
||||
out.Took = time.Since(start)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
var resp struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
Text string `json:"text"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
if json.Unmarshal(reply, &resp) == nil && len(resp.Choices) > 0 {
|
||||
out.Raw = strings.TrimSpace(resp.Choices[0].Message.Content + resp.Choices[0].Text)
|
||||
}
|
||||
res, err := parseJudgment(format, names, reply, answer)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Unsafe, out.Controversial, out.Categories = res.unsafe, res.controversial, res.categories
|
||||
return out, nil
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
)
|
||||
@@ -41,6 +42,11 @@ func (f *fakeJudge) call(ctx context.Context, _, _ string, body []byte) ([]byte,
|
||||
answer = map[bool]string{true: "Yes", false: "No"}[unsafe]
|
||||
case "json":
|
||||
answer = map[bool]string{true: `{"safe": false, "categories": ["violence"]}`, false: `{"safe": true}`}[unsafe]
|
||||
case "qwen3guard":
|
||||
answer = map[bool]string{true: "Safety: Unsafe\nCategories: Violent", false: "Safety: Safe\nCategories: None"}[unsafe]
|
||||
if strings.Contains(string(body), "a heist film") {
|
||||
answer = "Safety: Controversial\nCategories: Non-violent Illegal Acts"
|
||||
}
|
||||
default:
|
||||
answer = map[bool]string{true: "unsafe\nS1,S10", false: "safe"}[unsafe]
|
||||
}
|
||||
@@ -61,7 +67,7 @@ func judged(t *testing.T, p Policy, fj *fakeJudge) *Gate {
|
||||
|
||||
func TestAJudgeModelDecidesWhatPatternsCannot(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, format := range []string{"llamaguard3", "shieldgemma", "json"} {
|
||||
for _, format := range Formats {
|
||||
fj := &fakeJudge{format: format}
|
||||
g := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "block"}}, fj)
|
||||
if v, err := g.Input(ctx, Subject{}, chat("user", "plan an attack")); err != nil || v.Action != Block || v.Category != "judge" {
|
||||
@@ -191,3 +197,123 @@ func TestAnAnswerIsJudgedByLlamaGuardsOwnPrompt(t *testing.T) {
|
||||
t.Errorf("a completion's text: %+v %v", j, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Qwen3Guard's middle verdict counts as the policy says: safe unless it says
|
||||
// unsafe.
|
||||
func TestAControversialVerdictCountsAsThePolicySays(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
lenient := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "block"}}, &fakeJudge{format: "qwen3guard"})
|
||||
if v, _ := lenient.Input(ctx, Subject{}, chat("user", "the plot of a heist film")); v.Action != Allow {
|
||||
t.Errorf("controversial under the default: %+v", v)
|
||||
}
|
||||
strict := judged(t, Policy{Judge: &JudgePolicy{Model: "guard", Input: "block", Controversial: "unsafe"}}, &fakeJudge{format: "qwen3guard"})
|
||||
if v, _ := strict.Input(ctx, Subject{}, chat("user", "the plot of a heist film")); v.Action != Block || v.Rule != "Non-violent Illegal Acts" {
|
||||
t.Errorf("controversial: unsafe: %+v", v)
|
||||
}
|
||||
if err := (&JudgePolicy{Model: "guard", Input: "block", Controversial: "maybe"}).validate(); err == nil {
|
||||
t.Error("controversial: maybe was taken")
|
||||
}
|
||||
}
|
||||
|
||||
// A judge on a GPU first and one on CPUs after: the first that answers
|
||||
// decides, and the one that did not is said to have failed.
|
||||
func TestJudgesAreAskedInTurn(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var asked []string
|
||||
gpuDown := errors.New("no replica of gpu-judge is serving")
|
||||
g := gate(t, lib(map[string]Policy{"p": {Fail: "closed",
|
||||
Judge: &JudgePolicy{Models: []string{"gpu-judge", "cpu-judge"}, Input: "block"}}}, "p"))
|
||||
g.SetJudge(func(_ context.Context, model, _ string, body []byte) ([]byte, error) {
|
||||
asked = append(asked, model)
|
||||
if model == "gpu-judge" {
|
||||
return nil, gpuDown
|
||||
}
|
||||
unsafe := strings.Contains(string(body), "attack")
|
||||
answer := map[bool]string{true: "Safety: Unsafe\nCategories: Violent", false: "Safety: Safe\nCategories: None"}[unsafe]
|
||||
return json.Marshal(map[string]any{"choices": []any{map[string]any{"text": answer}}})
|
||||
}, func(m string) (string, []string, bool) { return "qwen3guard", nil, true })
|
||||
var alarmed []string
|
||||
g.SetJudgeAlarm(func(model string, _ error) { alarmed = append(alarmed, model) })
|
||||
|
||||
if v, err := g.Input(ctx, Subject{}, chat("user", "plan an attack")); err != nil || v.Action != Block || v.Rule != "Violent" {
|
||||
t.Errorf("unsafe: %+v %v", v, err)
|
||||
}
|
||||
if v, err := g.Input(ctx, Subject{}, chat("user", "continue")); err != nil || v.Action != Allow {
|
||||
t.Errorf("safe: %+v %v", v, err)
|
||||
}
|
||||
if !slices.Equal(asked, []string{"gpu-judge", "cpu-judge", "gpu-judge", "cpu-judge"}) {
|
||||
t.Errorf("asked %v", asked)
|
||||
}
|
||||
if !slices.Equal(alarmed, []string{"gpu-judge", "gpu-judge"}) {
|
||||
t.Errorf("alarm told %v", alarmed)
|
||||
}
|
||||
if e := g.JudgeErrors()["gpu-judge"]; !strings.Contains(e.Err, "no replica") {
|
||||
t.Errorf("the GPU judge's failure: %+v", e)
|
||||
}
|
||||
|
||||
// Every judge down: the policy's fail applies, as with one.
|
||||
g.SetJudge(func(context.Context, string, string, []byte) ([]byte, error) { return nil, gpuDown },
|
||||
func(string) (string, []string, bool) { return "qwen3guard", nil, true })
|
||||
if _, err := g.Input(ctx, Subject{}, chat("user", "hello")); err == nil {
|
||||
t.Error("fail closed served what no judge read")
|
||||
}
|
||||
|
||||
for _, bad := range []JudgePolicy{
|
||||
{Model: "a", Models: []string{"b"}, Input: "block"},
|
||||
{Models: []string{"a", "a"}, Input: "block"},
|
||||
{Models: []string{""}, Input: "block"},
|
||||
} {
|
||||
if err := bad.validate(); err == nil {
|
||||
t.Errorf("%+v was taken", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestQwen3GuardIsAskedInItsOwnWords(t *testing.T) {
|
||||
path, body := judgeRequest("qg", "qwen3guard", nil, "continue", "")
|
||||
var req struct{ Prompt string }
|
||||
_ = json.Unmarshal(body, &req)
|
||||
if path != "/v1/completions" || !strings.HasPrefix(req.Prompt, "<|im_start|>user\n# Task:\n1) Evaluate whether THE LAST USER's query") ||
|
||||
!strings.Contains(req.Prompt, "Jailbreak.\n<END UNSAFE CONTENT CATEGORIES>\n\n<BEGIN CONVERSATION>\n\nUSER: continue\n\n<END CONVERSATION>") ||
|
||||
!strings.HasSuffix(req.Prompt, "'Categories: None'.<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n") {
|
||||
t.Errorf("a request: %s %q", path, req.Prompt)
|
||||
}
|
||||
_, body = judgeRequest("qg", "qwen3guard", nil, "how do I bake bread", "Mix flour and water.")
|
||||
_ = json.Unmarshal(body, &req)
|
||||
if strings.Contains(req.Prompt, "Jailbreak") || !strings.Contains(req.Prompt, "<END REFUSAL POLICY>\n\n<BEGIN CONVERSATION>") ||
|
||||
!strings.Contains(req.Prompt, "USER: how do I bake bread\n\nASSISTANT: Mix flour and water.\n\n<END CONVERSATION>") ||
|
||||
!strings.Contains(req.Prompt, "'Categories: None'.\n - The third line must be one of: 'Refusal: Yes', 'Refusal: No'.<|im_end|>") {
|
||||
t.Errorf("an answer: %q", req.Prompt)
|
||||
}
|
||||
// An answer's verdict has a line on refusing that says nothing of safety.
|
||||
j, err := parseJudgment("qwen3guard", nil, []byte(`{"choices":[{"text":"Safety: Unsafe\nCategories: Violent, Jailbreak\nRefusal: No"}]}`), "")
|
||||
if err != nil || !j.unsafe || !slices.Equal(j.categories, []string{"Violent", "Jailbreak"}) {
|
||||
t.Errorf("%+v %v", j, err)
|
||||
}
|
||||
if j, err := parseJudgment("qwen3guard", nil, []byte(`{"choices":[{"text":"Safety: Safe\nCategories: None\nRefusal: Yes"}]}`), ""); err != nil || j.unsafe || len(j.categories) > 0 {
|
||||
t.Errorf("%+v %v", j, err)
|
||||
}
|
||||
if _, err := parseJudgment("qwen3guard", nil, []byte(`{"choices":[{"text":"I think it is fine"}]}`), ""); err == nil {
|
||||
t.Error("an answer with no verdict was read as one")
|
||||
}
|
||||
}
|
||||
|
||||
// A message with a file pasted into it is read by its start and its end, so
|
||||
// it fits the judge's context instead of overflowing it.
|
||||
func TestALongTextIsJudgedByItsEnds(t *testing.T) {
|
||||
file := strings.Repeat("ğ", 10000) // two bytes a rune: a cut must not split one
|
||||
msg := "review this file:\n" + file + "\nand plan an attack"
|
||||
_, body := judgeRequest("qg", "qwen3guard", nil, msg, "")
|
||||
var req struct{ Prompt string }
|
||||
_ = json.Unmarshal(body, &req)
|
||||
if !strings.Contains(req.Prompt, "USER: review this file:") || !strings.Contains(req.Prompt, "and plan an attack\n\n<END CONVERSATION>") ||
|
||||
!strings.Contains(req.Prompt, "\n[…]\n") || len(req.Prompt) > judgeHead+judgeTail+3000 {
|
||||
t.Errorf("%d bytes", len(req.Prompt))
|
||||
}
|
||||
if !utf8.ValidString(req.Prompt) {
|
||||
t.Error("a rune was cut in two")
|
||||
}
|
||||
if got := ends("short", 10, 10); got != "short" {
|
||||
t.Errorf("a short text was cut: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -193,8 +193,8 @@ func (f *outFilter) askJudge(text string, final bool) finding {
|
||||
res, err := f.g.judger.ask(f.ctx, jp, f.user, f.sofar.String())
|
||||
f.took += time.Since(start)
|
||||
if err != nil {
|
||||
f.g.log.Warn("the judge gave no verdict on an answer", "judge", jp.Model, "err", err)
|
||||
f.g.judgeFailed(jp.Model, err)
|
||||
f.g.log.Warn("the judge gave no verdict on an answer", "judge", jp.Name(), "err", err)
|
||||
f.g.noJudger(jp, err)
|
||||
}
|
||||
jf := judgeFinding(f.p, jp.Output, res, err)
|
||||
if jf.failOpen {
|
||||
|
||||
Loaded 100 of 175 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user