Notifications: webhooks for a parked model, a lost node, a spent budget and new weights; Alertmanager receiver templates

notify.webhooks sends signed JSON or Slack-compatible messages, queued,
retried, de-duplicated hourly; ctl notify test checks each webhook.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 16:58:11 +02:00
1 parent 646ff996c7
commit 1b71c38f89
24 files changed
+907 -4

No files matched your search

+11
View File
@@ -284,6 +284,7 @@ var Reference = []Block{
{Key: "engine_overrides", Type: "[]EngineOverride", Doc: "EngineOverrides give some nodes different engines from the rest: a\nvLLM built for CUDA 12 on the Volta cards CUDA 13 dropped, or none at\nall on nodes meant only for llama.cpp. Each picks nodes and lays its\nengines over the cluster's, in order, later ones last; an empty version\nremoves that engine from the nodes it picks.", Nested: ""},
{Key: "backup_dir", Type: "string", Doc: "BackupDir is where `llmgw server backup` writes, and where the newest\narchive's age is read for llmgw_backup_last_success_timestamp_seconds.\nEmpty means /var/backups/llmgw-server.", Nested: ""},
{Key: "guard", Type: "guard.Config", Doc: "Guard is the library of guardrail policies: what requests and answers\nare checked for, and what is done. Applied to every request through\nguard.default, and to parts of the tenancy tree by attaching them\n(llmgw ctl guard attach). Empty checks nothing.", Nested: "guard.Config"},
{Key: "notify", Type: "notify.Config", Doc: "Notify sends what needs a person — a model parked, a node lost, a\nbudget spent, new weights published — to webhooks, signed, without a\nPrometheus and Alertmanager in between. Empty sends nothing.", Nested: "notify.Config"},
{Key: "firewall_networks", Type: "[]string", Doc: "FirewallNetworks are networks, besides those each machine reaches the\ncluster on, that llmgw's internal ports answer: a monitoring host on\nanother subnet, nodes behind a router. Written into every machine's\nfirewall by `server init`, `node add` and `server add`.", Nested: ""},
{Key: "backup_recipients", Type: "[]string", Doc: "BackupRecipients encrypt every archive `llmgw server backup` writes, to\nwhoever holds one of their keys: age recipients (age1…) or SSH public\nkeys (ssh-ed25519 …, ssh-rsa …), as `age` takes them. An archive holds\nthe certificate authority's key and every secret in the store, and is\ncopied off the machine; encrypted, the copy is safe wherever it lands.\nEmpty writes archives unencrypted, readable by root only.", Nested: ""},
{Key: "releases", Type: "string", Doc: "Releases is where `llmgw ctl upgrade` fetches release binaries from,\nfor this machine and — through their agents — every node. Empty means\nthe project's own releases.", Nested: ""},
@@ -323,6 +324,12 @@ var Reference = []Block{
{Key: "launch_mode", Type: "string", Doc: "LaunchMode selects the multi-node mechanism: \"mp\", the default, or\n\"ray\".\n\nmp is what vLLM documents today — every node runs serve with its rank,\nthe workers headless — and needs nothing but vLLM. ray is the older way,\nkept for a vLLM too old for mp's multi-node flags; it needs Ray on every\nnode and has not been checked against a real cluster.", Nested: ""},
{Key: "ray_bin", Type: "string", Doc: "RayBin is used when LaunchMode is \"ray\".", Nested: ""},
}},
{Package: "notify", Name: "Webhook", Title: "Webhook", Doc: "", Fields: []Field{
{Key: "url", Type: "string", Doc: "", Nested: ""},
{Key: "format", Type: "string", Doc: "Format is json (the event, with X-Llmgw-Signature when SecretFile is\nset) or slack ({\"text\": …}, which Slack, Mattermost and Rocket.Chat\nincoming webhooks take).", Nested: ""},
{Key: "events", Type: "[]string", Doc: "Events are the kinds sent here; empty is all.", Nested: ""},
{Key: "secret_file", Type: "string", Doc: "SecretFile holds the key the body is signed with: X-Llmgw-Signature\nis sha256=<hex HMAC of the body>.", Nested: ""},
}},
{Package: "accounting", Name: "Price", Title: "accounting.Price", Doc: "", Fields: []Field{
{Key: "prompt_per_1k", Type: "int64", Doc: "", Nested: ""},
{Key: "completion_per_1k", Type: "int64", Doc: "", Nested: ""},
@@ -381,6 +388,10 @@ var Reference = []Block{
{Key: "default", Type: "[]string", Doc: "", Nested: ""},
{Key: "policies", Type: "map[string]Policy", Doc: "", Nested: ""},
}},
{Package: "notify", Name: "Config", Title: "notify.Config", Doc: "", Fields: []Field{
{Key: "webhooks", Type: "[]Webhook", Doc: "", Nested: ""},
{Key: "host", Type: "string", Doc: "Host names the gateway in messages; empty is this machine's name.", Nested: ""},
}},
{Package: "supervisor", Name: "Config", Title: "supervisor.Config", Doc: "", Fields: []Field{
{Key: "work_dir", Type: "string", Doc: "WorkDir holds pid files and per-shard scratch space.", Nested: ""},
{Key: "log_lines", Type: "int", Doc: "LogLines is how many lines of engine output to retain per shard.", Nested: ""},
+1
View File
@@ -29,6 +29,7 @@ var Sources = []string{
"internal/guard/policy.go",
"internal/guard/judge.go",
"internal/config/oidc.go",
"internal/notify/notify.go",
}
// notReference are files with koanf tags that an operator never writes.