Notifications: webhooks for a parked model, a lost node, a spent budget and new weights; Alertmanager receiver templates
notify.webhooks sends signed JSON or Slack-compatible messages, queued, retried, de-duplicated hourly; ctl notify test checks each webhook. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
646ff996c7
commit
1b71c38f89
24 files changed
+907
-4
No files matched your search
@@ -162,6 +162,7 @@ func newCtlCmd() *cobra.Command {
|
||||
newQuotaCmd(client, &asJSON),
|
||||
newGuardCmd(client, &asJSON),
|
||||
newAuditCmd(client, &asJSON),
|
||||
newNotifyCmd(client, &asJSON),
|
||||
newUsageCmd(client, &asJSON),
|
||||
newCertCmd(),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
)
|
||||
|
||||
func newNotifyCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
c := &cobra.Command{
|
||||
Use: "notify",
|
||||
Short: "The webhooks told about parked models, lost nodes, spent budgets and new weights",
|
||||
}
|
||||
c.AddCommand(&cobra.Command{
|
||||
Use: "test",
|
||||
Short: "Send a test notification to every webhook in server.yaml's notify section",
|
||||
Long: "Sends one event of kind \"test\" to each webhook, once, and says how each\n" +
|
||||
"answered: the way to check an address, a secret and a firewall before\n" +
|
||||
"something real has to travel.",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
var res []notify.Result
|
||||
if err := client().post(cmd.Context(), "/api/v1/notify/test", nil, &res); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(res)
|
||||
}
|
||||
failed := 0
|
||||
for _, r := range res {
|
||||
if r.Err != "" {
|
||||
failed++
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s failed: %s\n", r.URL, r.Err)
|
||||
} else {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s delivered\n", r.URL)
|
||||
}
|
||||
}
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d of %d webhooks were not reached", failed, len(res))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
return c
|
||||
}
|
||||
+14
-1
@@ -44,6 +44,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
|
||||
@@ -153,6 +154,16 @@ func newServerCmd() *cobra.Command {
|
||||
rt.Start(ctx)
|
||||
defer rt.Stop()
|
||||
|
||||
// Webhooks for what needs a person, when server.yaml names any.
|
||||
notifier, err := notify.New(ctx, cfg.Notify, log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rt.SetNotifier(notifier)
|
||||
if notifier != nil {
|
||||
log.Info("notifications on", "webhooks", len(cfg.Notify.Webhooks))
|
||||
}
|
||||
|
||||
// The control plane replaces the static endpoint list: agents
|
||||
// register, the reconciler places replicas, and Ready ones are
|
||||
// published to the router (plan §6.4).
|
||||
@@ -180,6 +191,7 @@ func newServerCmd() *cobra.Command {
|
||||
return err
|
||||
}
|
||||
defer ctl.Close()
|
||||
ctl.Reconciler().SetNotifier(notifier)
|
||||
admin = ctl
|
||||
agentPath, agentHandler = svc.Handler()
|
||||
|
||||
@@ -271,6 +283,7 @@ func newServerCmd() *cobra.Command {
|
||||
}
|
||||
adminAPI = api
|
||||
api.SetReleases(cfg.Releases)
|
||||
api.SetNotifier(notifier)
|
||||
if st := rt.Store(); st != nil && gate != nil {
|
||||
api.SetGuard(&adminapi.GuardAdmin{Library: cfg.Guard, Store: st, Invalidate: gate.Invalidate})
|
||||
}
|
||||
@@ -383,7 +396,7 @@ func newServerCmd() *cobra.Command {
|
||||
// changes it, and what replaces replicas.
|
||||
Current: func() []catalog.Model { return admin.Catalog().Models() },
|
||||
Models: catalogSvc, Restart: admin.Reconciler().Restart,
|
||||
Pool: adminAPI.PoolNodes,
|
||||
Pool: adminAPI.PoolNodes, Notify: notifier,
|
||||
})
|
||||
adminAPI.SetServe(serve)
|
||||
updateMetrics.Set(serve)
|
||||
|
||||
Reference in new issue
Block a user