Notifications: webhooks for a parked model, a lost node, a spent budget and new weights; Alertmanager receiver templates
notify.webhooks sends signed JSON or Slack-compatible messages, queued, retried, de-duplicated hourly; ctl notify test checks each webhook. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
646ff996c7
commit
1b71c38f89
24 files changed
+907
-4
No files matched your search
@@ -0,0 +1,288 @@
|
||||
// Package notify tells the people running the gateway about what needs them,
|
||||
// without a Prometheus in between: a model parked after failing every start,
|
||||
// a node lost, a budget spent, a model's repository with new weights. Each
|
||||
// event goes to the webhooks configured for it — JSON signed with a shared
|
||||
// secret, or a Slack-compatible message.
|
||||
//
|
||||
// Sending never holds up what raised the event: events are queued, and a
|
||||
// webhook that does not answer is retried a few times and then given up on.
|
||||
// The same event about the same thing is sent once an hour at most.
|
||||
package notify
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Event kinds.
|
||||
const (
|
||||
ModelParked = "model_parked"
|
||||
NodeLost = "node_lost"
|
||||
BudgetSpent = "budget_spent"
|
||||
UpdateAvailable = "update_available"
|
||||
)
|
||||
|
||||
// Kinds are every event there is.
|
||||
var Kinds = []string{ModelParked, NodeLost, BudgetSpent, UpdateAvailable}
|
||||
|
||||
// Event is one thing that happened.
|
||||
type Event struct {
|
||||
Kind string `json:"kind"`
|
||||
At time.Time `json:"at"`
|
||||
Subject string `json:"subject"`
|
||||
Message string `json:"message"`
|
||||
Detail map[string]any `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// Webhook is where events go.
|
||||
type Webhook struct {
|
||||
URL string `koanf:"url"`
|
||||
// Format is json (the event, with X-Llmgw-Signature when SecretFile is
|
||||
// set) or slack ({"text": …}, which Slack, Mattermost and Rocket.Chat
|
||||
// incoming webhooks take).
|
||||
Format string `koanf:"format"`
|
||||
// Events are the kinds sent here; empty is all.
|
||||
Events []string `koanf:"events"`
|
||||
// SecretFile holds the key the body is signed with: X-Llmgw-Signature
|
||||
// is sha256=<hex HMAC of the body>.
|
||||
SecretFile string `koanf:"secret_file"`
|
||||
}
|
||||
|
||||
// Config is server.yaml's notify: section.
|
||||
type Config struct {
|
||||
Webhooks []Webhook `koanf:"webhooks"`
|
||||
// Host names the gateway in messages; empty is this machine's name.
|
||||
Host string `koanf:"host"`
|
||||
}
|
||||
|
||||
// Validate checks the settings.
|
||||
func (c Config) Validate() error {
|
||||
for i, w := range c.Webhooks {
|
||||
if !strings.HasPrefix(w.URL, "https://") && !strings.HasPrefix(w.URL, "http://") {
|
||||
return fmt.Errorf("notify.webhooks[%d].url %q: an http(s) address", i, w.URL)
|
||||
}
|
||||
switch w.Format {
|
||||
case "", "json", "slack":
|
||||
default:
|
||||
return fmt.Errorf("notify.webhooks[%d].format %q: json or slack", i, w.Format)
|
||||
}
|
||||
for _, e := range w.Events {
|
||||
if !slices.Contains(Kinds, e) {
|
||||
return fmt.Errorf("notify.webhooks[%d].events: %q is not one of %s", i, e, strings.Join(Kinds, ", "))
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Notifier sends events. A nil one sends nothing.
|
||||
type Notifier struct {
|
||||
hooks []hook
|
||||
host string
|
||||
log *slog.Logger
|
||||
client *http.Client
|
||||
queue chan Event
|
||||
retry time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
sent map[string]time.Time
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
type hook struct {
|
||||
Webhook
|
||||
secret []byte
|
||||
}
|
||||
|
||||
// dedupe is how long the same event about the same thing is not sent again.
|
||||
const dedupe = time.Hour
|
||||
|
||||
// New builds a notifier and starts sending; it stops when ctx ends. Nil when
|
||||
// nothing is configured.
|
||||
func New(ctx context.Context, c Config, log *slog.Logger) (*Notifier, error) {
|
||||
if len(c.Webhooks) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if err := c.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
n := &Notifier{host: c.Host, log: log, client: &http.Client{Timeout: 10 * time.Second},
|
||||
queue: make(chan Event, 256), sent: map[string]time.Time{}, retry: 5 * time.Second}
|
||||
if n.host == "" {
|
||||
n.host, _ = os.Hostname()
|
||||
}
|
||||
for _, w := range c.Webhooks {
|
||||
h := hook{Webhook: w}
|
||||
if w.SecretFile != "" {
|
||||
b, err := os.ReadFile(w.SecretFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("notify: %w", err)
|
||||
}
|
||||
h.secret = bytes.TrimSpace(b)
|
||||
}
|
||||
n.hooks = append(n.hooks, h)
|
||||
}
|
||||
n.wg.Add(1)
|
||||
go n.run(ctx)
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Notify queues an event. It never blocks: with the queue full the event is
|
||||
// dropped, and said so in the log.
|
||||
func (n *Notifier) Notify(kind, subject, message string, detail map[string]any) {
|
||||
if n == nil {
|
||||
return
|
||||
}
|
||||
k := kind + "\x00" + subject
|
||||
now := time.Now()
|
||||
n.mu.Lock()
|
||||
if last, ok := n.sent[k]; ok && now.Sub(last) < dedupe {
|
||||
n.mu.Unlock()
|
||||
return
|
||||
}
|
||||
n.sent[k] = now
|
||||
for key, at := range n.sent {
|
||||
if now.Sub(at) > dedupe {
|
||||
delete(n.sent, key)
|
||||
}
|
||||
}
|
||||
n.mu.Unlock()
|
||||
select {
|
||||
case n.queue <- Event{Kind: kind, At: now.UTC(), Subject: subject, Message: message, Detail: detail}:
|
||||
default:
|
||||
n.log.Warn("a notification was dropped: too many waiting", "kind", kind, "subject", subject)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait waits for what is queued to be sent, for tests and shutdown.
|
||||
func (n *Notifier) Wait() {
|
||||
if n == nil {
|
||||
return
|
||||
}
|
||||
close(n.queue)
|
||||
n.wg.Wait()
|
||||
}
|
||||
|
||||
func (n *Notifier) run(ctx context.Context) {
|
||||
defer n.wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-n.queue:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
for _, h := range n.hooks {
|
||||
if len(h.Events) > 0 && !slices.Contains(h.Events, e.Kind) {
|
||||
continue
|
||||
}
|
||||
if err := n.send(ctx, h, e, 3); err != nil {
|
||||
n.log.Warn("a notification could not be delivered", "kind", e.Kind, "subject", e.Subject, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test sends a test event to every webhook now, once each, and says how each
|
||||
// answered: what `llmgw ctl notify test` shows.
|
||||
func (n *Notifier) Test(ctx context.Context) []Result {
|
||||
if n == nil {
|
||||
return nil
|
||||
}
|
||||
e := Event{Kind: "test", At: time.Now().UTC(), Subject: "test",
|
||||
Message: "a test notification: this webhook is reached"}
|
||||
var out []Result
|
||||
for _, h := range n.hooks {
|
||||
r := Result{URL: redact(h.URL)}
|
||||
if err := n.send(ctx, h, e, 1); err != nil {
|
||||
r.Err = err.Error()
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Result is how one webhook answered a test.
|
||||
type Result struct {
|
||||
URL string `json:"url"`
|
||||
Err string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// redact keeps a webhook's host and drops its path, where Slack and others
|
||||
// put the secret.
|
||||
func redact(u string) string {
|
||||
if i := strings.Index(u, "://"); i >= 0 {
|
||||
if j := strings.IndexByte(u[i+3:], '/'); j >= 0 {
|
||||
return u[:i+3+j] + "/…"
|
||||
}
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
func (n *Notifier) send(ctx context.Context, h hook, e Event, attempts int) error {
|
||||
var body []byte
|
||||
if h.Format == "slack" {
|
||||
body, _ = json.Marshal(map[string]string{"text": fmt.Sprintf("*llmgw on %s* — %s", n.host, e.Message)})
|
||||
} else {
|
||||
ev := e
|
||||
if ev.Detail == nil {
|
||||
ev.Detail = map[string]any{}
|
||||
}
|
||||
ev.Detail["gateway"] = n.host
|
||||
body, _ = json.Marshal(ev)
|
||||
}
|
||||
for attempt := 1; ; attempt++ {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, h.URL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-Llmgw-Event", e.Kind)
|
||||
if h.secret != nil {
|
||||
m := hmac.New(sha256.New, h.secret)
|
||||
m.Write(body)
|
||||
req.Header.Set("X-Llmgw-Signature", "sha256="+hex.EncodeToString(m.Sum(nil)))
|
||||
}
|
||||
resp, err := n.client.Do(req)
|
||||
// Not the error as it comes: it names the whole address, and the
|
||||
// path of a Slack or PagerDuty webhook is its secret.
|
||||
var ue *url.Error
|
||||
if errors.As(err, &ue) {
|
||||
err = fmt.Errorf("%s %s: %w", ue.Op, redact(h.URL), ue.Err)
|
||||
}
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode < 300 {
|
||||
return nil
|
||||
}
|
||||
err = fmt.Errorf("answered %s", resp.Status)
|
||||
}
|
||||
if attempt >= attempts {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(n.retry * time.Duration(attempt)):
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user