Notifications: webhooks for a parked model, a lost node, a spent budget and new weights; Alertmanager receiver templates

notify.webhooks sends signed JSON or Slack-compatible messages, queued,
retried, de-duplicated hourly; ctl notify test checks each webhook.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 16:58:11 +02:00
1 parent 646ff996c7
commit 1b71c38f89
24 files changed
+907 -4

No files matched your search

+1
View File
@@ -162,6 +162,7 @@ func newCtlCmd() *cobra.Command {
newQuotaCmd(client, &asJSON),
newGuardCmd(client, &asJSON),
newAuditCmd(client, &asJSON),
newNotifyCmd(client, &asJSON),
newUsageCmd(client, &asJSON),
newCertCmd(),
)
+48
View File
@@ -0,0 +1,48 @@
package cli
import (
"encoding/json"
"fmt"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
)
func newNotifyCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
c := &cobra.Command{
Use: "notify",
Short: "The webhooks told about parked models, lost nodes, spent budgets and new weights",
}
c.AddCommand(&cobra.Command{
Use: "test",
Short: "Send a test notification to every webhook in server.yaml's notify section",
Long: "Sends one event of kind \"test\" to each webhook, once, and says how each\n" +
"answered: the way to check an address, a secret and a firewall before\n" +
"something real has to travel.",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
var res []notify.Result
if err := client().post(cmd.Context(), "/api/v1/notify/test", nil, &res); err != nil {
return err
}
if *asJSON {
return json.NewEncoder(cmd.OutOrStdout()).Encode(res)
}
failed := 0
for _, r := range res {
if r.Err != "" {
failed++
fmt.Fprintf(cmd.OutOrStdout(), "%s failed: %s\n", r.URL, r.Err)
} else {
fmt.Fprintf(cmd.OutOrStdout(), "%s delivered\n", r.URL)
}
}
if failed > 0 {
return fmt.Errorf("%d of %d webhooks were not reached", failed, len(res))
}
return nil
},
})
return c
}
+14 -1
View File
@@ -44,6 +44,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
@@ -153,6 +154,16 @@ func newServerCmd() *cobra.Command {
rt.Start(ctx)
defer rt.Stop()
// Webhooks for what needs a person, when server.yaml names any.
notifier, err := notify.New(ctx, cfg.Notify, log)
if err != nil {
return err
}
rt.SetNotifier(notifier)
if notifier != nil {
log.Info("notifications on", "webhooks", len(cfg.Notify.Webhooks))
}
// The control plane replaces the static endpoint list: agents
// register, the reconciler places replicas, and Ready ones are
// published to the router (plan §6.4).
@@ -180,6 +191,7 @@ func newServerCmd() *cobra.Command {
return err
}
defer ctl.Close()
ctl.Reconciler().SetNotifier(notifier)
admin = ctl
agentPath, agentHandler = svc.Handler()
@@ -271,6 +283,7 @@ func newServerCmd() *cobra.Command {
}
adminAPI = api
api.SetReleases(cfg.Releases)
api.SetNotifier(notifier)
if st := rt.Store(); st != nil && gate != nil {
api.SetGuard(&adminapi.GuardAdmin{Library: cfg.Guard, Store: st, Invalidate: gate.Invalidate})
}
@@ -383,7 +396,7 @@ func newServerCmd() *cobra.Command {
// changes it, and what replaces replicas.
Current: func() []catalog.Model { return admin.Catalog().Models() },
Models: catalogSvc, Restart: admin.Reconciler().Restart,
Pool: adminAPI.PoolNodes,
Pool: adminAPI.PoolNodes, Notify: notifier,
})
adminAPI.SetServe(serve)
updateMetrics.Set(serve)