Status says when the cluster's alerts go nowhere
CI / test (push) Failing after 6m57s

Setup asks for webhooks, but one left empty was never mentioned again: a
lost node or a judge down was told nobody. With no notify.webhooks in the
cluster's settings, ctl status and the console's Overview say so, and how
to set one. BACKLOG: the judge's refusal after a placement change, tried
again on idle VMs, did not happen; a cold judge's first answer is 2-3x
slower but inside its budget.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-12 22:49:37 +01:00
1 parent 2c32a58a06
commit a00c92648e
10 files changed
+60 -4

No files matched your search

+2
View File
@@ -100,6 +100,8 @@ type API struct {
ctl *control.Controller
// slurmUsers are the Linux users made from the accounts; nil without a store.
slurmUsers SlurmUsers
// webhooks counts the settings' notify.webhooks; nil does not say.
webhooks func() int
}
// New builds the API over a controller.
+4
View File
@@ -6,6 +6,10 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
)
// SetWebhooks has the summary say when the settings name no webhook:
// count is how many they name now.
func (a *API) SetWebhooks(count func() int) { a.webhooks = count }
// SetNotifier gives the API the webhooks `ctl notify test` reaches.
func (a *API) SetNotifier(n *notify.Notifier) { a.notifier = n }
+6
View File
@@ -50,6 +50,9 @@ type Summary struct {
// GuardOff says the guardrails are turned off (guard.enabled: false):
// no request is checked, whatever the policies say.
GuardOff bool `json:"guard_off,omitempty"`
// NoAlerts says the cluster's settings name no webhook: a model parked,
// a node lost or a judge down is told nobody.
NoAlerts bool `json:"no_alerts,omitempty"`
}
// StoreMember is one member of a replicated store, as its Patroni reports it.
@@ -160,6 +163,9 @@ func (a *API) Summary() Summary {
if a.guard != nil && a.guard.Library != nil && !a.guard.Library().On() {
s.GuardOff = true
}
if a.webhooks != nil && a.webhooks() == 0 {
s.NoAlerts = true
}
for _, j := range a.Judges() {
if j.Problem != "" {
s.Judges = append(s.Judges, j)
+13
View File
@@ -153,3 +153,16 @@ func TestOneSharedCopyCountsAsMuchAsTwenty(t *testing.T) {
t.Errorf("counted %d models; three nodes seeing one file is one model", f.Models)
}
}
func TestTheSummarySaysWhenNoWebhookIsSet(t *testing.T) {
a, _ := setup(t, testModels(), gpuNode("n1", 8))
n := 0
a.SetWebhooks(func() int { return n })
if !a.Summary().NoAlerts {
t.Error("no webhook, not said")
}
n = 1
if a.Summary().NoAlerts {
t.Error("a webhook set, yet said there is none")
}
}