diff --git a/BACKLOG.md b/BACKLOG.md index 063e9c5..dd96df3 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -463,8 +463,9 @@ portal is not done — the page shows; the console and ctl change. 5. **Done (branch `features-c`):** `notify.webhooks` (model parked, node lost, budget spent, update available; signed JSON or Slack; retries, hourly de-duplication), `ctl notify test`, Alertmanager receiver - templates. Left: `server init` and the console do not yet ask for a - receiver — the templates are filled in by hand. Events are claimed in + templates. `server init` and both setup screens ask for webhooks and test + each (d1e9186, 2026-10-14); with none set, `ctl status` and the console's + Overview say the alerts go nowhere (2026-11-12). Events are claimed in the store before they are sent (2026-10-10), so two control planes send each once. **Alerting out of the box (S).** `deploy/monitoring/alertmanager.yml` @@ -2056,7 +2057,11 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t node (the controller is its node), so a site wanting control planes without nodes chooses the controller later with `ctl slurm setup`; batch output lands on the job's first node until there is shared storage; on shared CPUs the judge refuses the first request after a placement change (fail - closed, ~10 s), and its advice repeats `--cores 4` to a judge that has 4. + closed, ~10 s), and its advice repeats `--cores 4` to a judge that has 4. Tried again on the idle + VMs: one judge restarted in place, then both at once, requests every second: 48 of 48 served; the + first after a cold start took 7.6–8.9 s end to end against ~3 s, within the 5 s judge budget. So the + refusal was the install's load on the CPUs; if it is seen again, warm a new judge with one judgment + before it takes requests. - Shared files and users, built 2026-11-12: `slurm.shared` {path, nfs | server} — the site's export mounted (a systemd mount unit, hard, nconnect=4), an llmgw node exporting the path itself (/etc/exports.d, to the Slurm machines' addresses, root_squash; 2049 in the firewall), or the site's diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 0a2df86..0386264 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -9595,6 +9595,10 @@ "guard_off": { "type": "boolean", "description": "GuardOff says the guardrails are turned off (guard.enabled: false): no request is checked, whatever the policies say." + }, + "no_alerts": { + "type": "boolean", + "description": "NoAlerts says the cluster's settings name no webhook: a model parked, a node lost or a judge down is told nobody." } } }, diff --git a/docs/console.md b/docs/console.md index 6baaf06..4c10878 100644 --- a/docs/console.md +++ b/docs/console.md @@ -67,7 +67,8 @@ or with the mirror unticked, is said in the form and nothing starts. ## Watch **Overview** is `llmgw ctl status`: nodes, GPUs by class, fabrics, replicas, -models, the store's members, and anything parked, unplaced or idle. +models, the store's members, and anything parked, unplaced or idle. With no +webhook in the cluster's settings it says the alerts go nowhere. **Monitor** draws the cluster the way btop draws a machine: CPU, memory, GPU busy and GPU memory, requests and tokens a second, time to first token, diff --git a/internal/cli/ctl_status_test.go b/internal/cli/ctl_status_test.go index e814d81..51ca898 100644 --- a/internal/cli/ctl_status_test.go +++ b/internal/cli/ctl_status_test.go @@ -84,3 +84,19 @@ func TestTheSummaryIsWrittenWhereItIsToldNotToTheTerminal(t *testing.T) { } } } + +func TestStatusSaysWhenAlertsGoNowhere(t *testing.T) { + render := func(s adminapi.Summary) string { + cmd := &cobra.Command{} + var b bytes.Buffer + cmd.SetOut(&b) + printSummary(cmd, s) + return b.String() + } + if out := render(adminapi.Summary{NoAlerts: true}); !strings.Contains(out, "ALERTS nowhere") { + t.Errorf("no webhook was not called out:\n%s", out) + } + if out := render(adminapi.Summary{}); strings.Contains(out, "ALERTS") { + t.Errorf("an alerts line with webhooks set:\n%s", out) + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 7ce7d4d..99e60fd 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -417,6 +417,7 @@ func newServerCmd() *cobra.Command { adminAPI = api api.SetReleases(cfg.Releases) api.SetNotifier(notifier) + api.SetWebhooks(func() int { return len(current.Load().Notify.Webhooks) }) api.SetReplayer(rt.Replay) // Quality suites, asked of a model the way a replay is. evals = newEvalService(cfg, rt, admin, log) diff --git a/internal/cli/status.go b/internal/cli/status.go index 3260bee..f05cec3 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -156,6 +156,10 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) { if s.Stopped { fmt.Fprintln(out, "SERVICE STOPPED: no model runs and requests are answered 503 (llmgw start)") } + if s.NoAlerts { + fmt.Fprintln(out, "ALERTS nowhere: a lost node or a judge down is told nobody "+ + "(llmgw ctl config set notify.webhooks, then llmgw ctl notify test)") + } if s.GuardOff { fmt.Fprintln(out, "GUARD off: no request is checked, whatever the policies say (llmgw ctl guard on)") } diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 43143cb..3af429c 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -100,6 +100,8 @@ type API struct { ctl *control.Controller // slurmUsers are the Linux users made from the accounts; nil without a store. slurmUsers SlurmUsers + // webhooks counts the settings' notify.webhooks; nil does not say. + webhooks func() int } // New builds the API over a controller. diff --git a/internal/control/adminapi/notify.go b/internal/control/adminapi/notify.go index 20a3b17..5d69428 100644 --- a/internal/control/adminapi/notify.go +++ b/internal/control/adminapi/notify.go @@ -6,6 +6,10 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/notify" ) +// SetWebhooks has the summary say when the settings name no webhook: +// count is how many they name now. +func (a *API) SetWebhooks(count func() int) { a.webhooks = count } + // SetNotifier gives the API the webhooks `ctl notify test` reaches. func (a *API) SetNotifier(n *notify.Notifier) { a.notifier = n } diff --git a/internal/control/adminapi/summary.go b/internal/control/adminapi/summary.go index 0a31539..976cc7d 100644 --- a/internal/control/adminapi/summary.go +++ b/internal/control/adminapi/summary.go @@ -50,6 +50,9 @@ type Summary struct { // GuardOff says the guardrails are turned off (guard.enabled: false): // no request is checked, whatever the policies say. GuardOff bool `json:"guard_off,omitempty"` + // NoAlerts says the cluster's settings name no webhook: a model parked, + // a node lost or a judge down is told nobody. + NoAlerts bool `json:"no_alerts,omitempty"` } // StoreMember is one member of a replicated store, as its Patroni reports it. @@ -160,6 +163,9 @@ func (a *API) Summary() Summary { if a.guard != nil && a.guard.Library != nil && !a.guard.Library().On() { s.GuardOff = true } + if a.webhooks != nil && a.webhooks() == 0 { + s.NoAlerts = true + } for _, j := range a.Judges() { if j.Problem != "" { s.Judges = append(s.Judges, j) diff --git a/internal/control/adminapi/summary_test.go b/internal/control/adminapi/summary_test.go index ecc7e8a..acc7bf7 100644 --- a/internal/control/adminapi/summary_test.go +++ b/internal/control/adminapi/summary_test.go @@ -153,3 +153,16 @@ func TestOneSharedCopyCountsAsMuchAsTwenty(t *testing.T) { t.Errorf("counted %d models; three nodes seeing one file is one model", f.Models) } } + +func TestTheSummarySaysWhenNoWebhookIsSet(t *testing.T) { + a, _ := setup(t, testModels(), gpuNode("n1", 8)) + n := 0 + a.SetWebhooks(func() int { return n }) + if !a.Summary().NoAlerts { + t.Error("no webhook, not said") + } + n = 1 + if a.Summary().NoAlerts { + t.Error("a webhook set, yet said there is none") + } +}