Notifications: webhooks for a parked model, a lost node, a spent budget and new weights; Alertmanager receiver templates

notify.webhooks sends signed JSON or Slack-compatible messages, queued,
retried, de-duplicated hourly; ctl notify test checks each webhook.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

internal/control/adminapi/api.go

@@ -35,11 +35,14 @@ import (
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
 )
 
 // API serves the management endpoints.
 type API struct {
+	// notifier is the webhooks `ctl notify test` reaches; nil without any.
+	notifier *notify.Notifier
 	// releases is where release binaries are fetched from, for upgrades.
 	releases string
 	// storeStatus reads a replicated store's members; nil for one database.
@@ -84,6 +87,7 @@ func (a *API) Register(mux *http.ServeMux) {
 	a.registerGuard(mux)
 	mux.HandleFunc("GET /api/v1/audit", a.auditEvents)
 	mux.HandleFunc("GET /api/v1/usage/export", a.usageExport)
+	mux.HandleFunc("POST /api/v1/notify/test", a.notifyTest)
 	mux.HandleFunc("POST /api/v1/models/{name}/adapters", a.setAdapters)
 	mux.HandleFunc("GET /api/v1/models/{name}/target", a.target)
 	mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) {

internal/control/adminapi/notify.go

@@ -0,0 +1,19 @@
+package adminapi
+
+import (
+	"net/http"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
+)
+
+// SetNotifier gives the API the webhooks `ctl notify test` reaches.
+func (a *API) SetNotifier(n *notify.Notifier) { a.notifier = n }
+
+// notifyTest sends a test event to every webhook and says how each answered.
+func (a *API) notifyTest(w http.ResponseWriter, r *http.Request) {
+	if a.notifier == nil {
+		writeError(w, http.StatusNotFound, "no webhooks: server.yaml has no notify.webhooks")
+		return
+	}
+	writeJSON(w, http.StatusOK, a.notifier.Test(r.Context()))
+}