Notifications: webhooks for a parked model, a lost node, a spent budget and new weights; Alertmanager receiver templates
notify.webhooks sends signed JSON or Slack-compatible messages, queued, retried, de-duplicated hourly; ctl notify test checks each webhook. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
646ff996c7
commit
1b71c38f89
24 files changed
+907
-4
No files matched your search
@@ -0,0 +1,288 @@
|
||||
// Package notify tells the people running the gateway about what needs them,
|
||||
// without a Prometheus in between: a model parked after failing every start,
|
||||
// a node lost, a budget spent, a model's repository with new weights. Each
|
||||
// event goes to the webhooks configured for it — JSON signed with a shared
|
||||
// secret, or a Slack-compatible message.
|
||||
//
|
||||
// Sending never holds up what raised the event: events are queued, and a
|
||||
// webhook that does not answer is retried a few times and then given up on.
|
||||
// The same event about the same thing is sent once an hour at most.
|
||||
package notify
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Event kinds.
|
||||
const (
|
||||
ModelParked = "model_parked"
|
||||
NodeLost = "node_lost"
|
||||
BudgetSpent = "budget_spent"
|
||||
UpdateAvailable = "update_available"
|
||||
)
|
||||
|
||||
// Kinds are every event there is.
|
||||
var Kinds = []string{ModelParked, NodeLost, BudgetSpent, UpdateAvailable}
|
||||
|
||||
// Event is one thing that happened.
|
||||
type Event struct {
|
||||
Kind string `json:"kind"`
|
||||
At time.Time `json:"at"`
|
||||
Subject string `json:"subject"`
|
||||
Message string `json:"message"`
|
||||
Detail map[string]any `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// Webhook is where events go.
|
||||
type Webhook struct {
|
||||
URL string `koanf:"url"`
|
||||
// Format is json (the event, with X-Llmgw-Signature when SecretFile is
|
||||
// set) or slack ({"text": …}, which Slack, Mattermost and Rocket.Chat
|
||||
// incoming webhooks take).
|
||||
Format string `koanf:"format"`
|
||||
// Events are the kinds sent here; empty is all.
|
||||
Events []string `koanf:"events"`
|
||||
// SecretFile holds the key the body is signed with: X-Llmgw-Signature
|
||||
// is sha256=<hex HMAC of the body>.
|
||||
SecretFile string `koanf:"secret_file"`
|
||||
}
|
||||
|
||||
// Config is server.yaml's notify: section.
|
||||
type Config struct {
|
||||
Webhooks []Webhook `koanf:"webhooks"`
|
||||
// Host names the gateway in messages; empty is this machine's name.
|
||||
Host string `koanf:"host"`
|
||||
}
|
||||
|
||||
// Validate checks the settings.
|
||||
func (c Config) Validate() error {
|
||||
for i, w := range c.Webhooks {
|
||||
if !strings.HasPrefix(w.URL, "https://") && !strings.HasPrefix(w.URL, "http://") {
|
||||
return fmt.Errorf("notify.webhooks[%d].url %q: an http(s) address", i, w.URL)
|
||||
}
|
||||
switch w.Format {
|
||||
case "", "json", "slack":
|
||||
default:
|
||||
return fmt.Errorf("notify.webhooks[%d].format %q: json or slack", i, w.Format)
|
||||
}
|
||||
for _, e := range w.Events {
|
||||
if !slices.Contains(Kinds, e) {
|
||||
return fmt.Errorf("notify.webhooks[%d].events: %q is not one of %s", i, e, strings.Join(Kinds, ", "))
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Notifier sends events. A nil one sends nothing.
|
||||
type Notifier struct {
|
||||
hooks []hook
|
||||
host string
|
||||
log *slog.Logger
|
||||
client *http.Client
|
||||
queue chan Event
|
||||
retry time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
sent map[string]time.Time
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
type hook struct {
|
||||
Webhook
|
||||
secret []byte
|
||||
}
|
||||
|
||||
// dedupe is how long the same event about the same thing is not sent again.
|
||||
const dedupe = time.Hour
|
||||
|
||||
// New builds a notifier and starts sending; it stops when ctx ends. Nil when
|
||||
// nothing is configured.
|
||||
func New(ctx context.Context, c Config, log *slog.Logger) (*Notifier, error) {
|
||||
if len(c.Webhooks) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if err := c.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
n := &Notifier{host: c.Host, log: log, client: &http.Client{Timeout: 10 * time.Second},
|
||||
queue: make(chan Event, 256), sent: map[string]time.Time{}, retry: 5 * time.Second}
|
||||
if n.host == "" {
|
||||
n.host, _ = os.Hostname()
|
||||
}
|
||||
for _, w := range c.Webhooks {
|
||||
h := hook{Webhook: w}
|
||||
if w.SecretFile != "" {
|
||||
b, err := os.ReadFile(w.SecretFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("notify: %w", err)
|
||||
}
|
||||
h.secret = bytes.TrimSpace(b)
|
||||
}
|
||||
n.hooks = append(n.hooks, h)
|
||||
}
|
||||
n.wg.Add(1)
|
||||
go n.run(ctx)
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Notify queues an event. It never blocks: with the queue full the event is
|
||||
// dropped, and said so in the log.
|
||||
func (n *Notifier) Notify(kind, subject, message string, detail map[string]any) {
|
||||
if n == nil {
|
||||
return
|
||||
}
|
||||
k := kind + "\x00" + subject
|
||||
now := time.Now()
|
||||
n.mu.Lock()
|
||||
if last, ok := n.sent[k]; ok && now.Sub(last) < dedupe {
|
||||
n.mu.Unlock()
|
||||
return
|
||||
}
|
||||
n.sent[k] = now
|
||||
for key, at := range n.sent {
|
||||
if now.Sub(at) > dedupe {
|
||||
delete(n.sent, key)
|
||||
}
|
||||
}
|
||||
n.mu.Unlock()
|
||||
select {
|
||||
case n.queue <- Event{Kind: kind, At: now.UTC(), Subject: subject, Message: message, Detail: detail}:
|
||||
default:
|
||||
n.log.Warn("a notification was dropped: too many waiting", "kind", kind, "subject", subject)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait waits for what is queued to be sent, for tests and shutdown.
|
||||
func (n *Notifier) Wait() {
|
||||
if n == nil {
|
||||
return
|
||||
}
|
||||
close(n.queue)
|
||||
n.wg.Wait()
|
||||
}
|
||||
|
||||
func (n *Notifier) run(ctx context.Context) {
|
||||
defer n.wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-n.queue:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
for _, h := range n.hooks {
|
||||
if len(h.Events) > 0 && !slices.Contains(h.Events, e.Kind) {
|
||||
continue
|
||||
}
|
||||
if err := n.send(ctx, h, e, 3); err != nil {
|
||||
n.log.Warn("a notification could not be delivered", "kind", e.Kind, "subject", e.Subject, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test sends a test event to every webhook now, once each, and says how each
|
||||
// answered: what `llmgw ctl notify test` shows.
|
||||
func (n *Notifier) Test(ctx context.Context) []Result {
|
||||
if n == nil {
|
||||
return nil
|
||||
}
|
||||
e := Event{Kind: "test", At: time.Now().UTC(), Subject: "test",
|
||||
Message: "a test notification: this webhook is reached"}
|
||||
var out []Result
|
||||
for _, h := range n.hooks {
|
||||
r := Result{URL: redact(h.URL)}
|
||||
if err := n.send(ctx, h, e, 1); err != nil {
|
||||
r.Err = err.Error()
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Result is how one webhook answered a test.
|
||||
type Result struct {
|
||||
URL string `json:"url"`
|
||||
Err string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// redact keeps a webhook's host and drops its path, where Slack and others
|
||||
// put the secret.
|
||||
func redact(u string) string {
|
||||
if i := strings.Index(u, "://"); i >= 0 {
|
||||
if j := strings.IndexByte(u[i+3:], '/'); j >= 0 {
|
||||
return u[:i+3+j] + "/…"
|
||||
}
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
func (n *Notifier) send(ctx context.Context, h hook, e Event, attempts int) error {
|
||||
var body []byte
|
||||
if h.Format == "slack" {
|
||||
body, _ = json.Marshal(map[string]string{"text": fmt.Sprintf("*llmgw on %s* — %s", n.host, e.Message)})
|
||||
} else {
|
||||
ev := e
|
||||
if ev.Detail == nil {
|
||||
ev.Detail = map[string]any{}
|
||||
}
|
||||
ev.Detail["gateway"] = n.host
|
||||
body, _ = json.Marshal(ev)
|
||||
}
|
||||
for attempt := 1; ; attempt++ {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, h.URL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-Llmgw-Event", e.Kind)
|
||||
if h.secret != nil {
|
||||
m := hmac.New(sha256.New, h.secret)
|
||||
m.Write(body)
|
||||
req.Header.Set("X-Llmgw-Signature", "sha256="+hex.EncodeToString(m.Sum(nil)))
|
||||
}
|
||||
resp, err := n.client.Do(req)
|
||||
// Not the error as it comes: it names the whole address, and the
|
||||
// path of a Slack or PagerDuty webhook is its secret.
|
||||
var ue *url.Error
|
||||
if errors.As(err, &ue) {
|
||||
err = fmt.Errorf("%s %s: %w", ue.Op, redact(h.URL), ue.Err)
|
||||
}
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode < 300 {
|
||||
return nil
|
||||
}
|
||||
err = fmt.Errorf("answered %s", resp.Status)
|
||||
}
|
||||
if attempt >= attempts {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(n.retry * time.Duration(attempt)):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type got struct {
|
||||
mu sync.Mutex
|
||||
calls []*http.Request
|
||||
body [][]byte
|
||||
}
|
||||
|
||||
func (g *got) handler(fail int) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.calls = append(g.calls, r)
|
||||
g.body = append(g.body, b)
|
||||
if len(g.calls) <= fail {
|
||||
w.WriteHeader(http.StatusBadGateway)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newNotifier(t *testing.T, c Config) *Notifier {
|
||||
t.Helper()
|
||||
n, err := New(context.Background(), c, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n.retry = time.Millisecond
|
||||
return n
|
||||
}
|
||||
|
||||
func TestSignedJSON(t *testing.T) {
|
||||
var g got
|
||||
srv := httptest.NewServer(g.handler(0))
|
||||
defer srv.Close()
|
||||
secret := filepath.Join(t.TempDir(), "s")
|
||||
os.WriteFile(secret, []byte("k3y\n"), 0o600)
|
||||
|
||||
n := newNotifier(t, Config{Host: "gw1", Webhooks: []Webhook{{URL: srv.URL, SecretFile: secret}}})
|
||||
n.Notify(ModelParked, "qwen", "model qwen is parked", map[string]any{"failures": 3})
|
||||
n.Wait()
|
||||
|
||||
if len(g.calls) != 1 {
|
||||
t.Fatalf("%d calls, want 1", len(g.calls))
|
||||
}
|
||||
m := hmac.New(sha256.New, []byte("k3y"))
|
||||
m.Write(g.body[0])
|
||||
if want := "sha256=" + hex.EncodeToString(m.Sum(nil)); g.calls[0].Header.Get("X-Llmgw-Signature") != want {
|
||||
t.Errorf("signature %q, want %q", g.calls[0].Header.Get("X-Llmgw-Signature"), want)
|
||||
}
|
||||
if g.calls[0].Header.Get("X-Llmgw-Event") != ModelParked {
|
||||
t.Errorf("event header %q", g.calls[0].Header.Get("X-Llmgw-Event"))
|
||||
}
|
||||
var e Event
|
||||
if err := json.Unmarshal(g.body[0], &e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e.Kind != ModelParked || e.Subject != "qwen" || e.Detail["gateway"] != "gw1" || e.Detail["failures"] != float64(3) {
|
||||
t.Errorf("event %+v", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlackFiltersAndDedupes(t *testing.T) {
|
||||
var g got
|
||||
srv := httptest.NewServer(g.handler(0))
|
||||
defer srv.Close()
|
||||
|
||||
n := newNotifier(t, Config{Host: "gw1", Webhooks: []Webhook{{URL: srv.URL, Format: "slack", Events: []string{NodeLost}}}})
|
||||
n.Notify(ModelParked, "qwen", "not for this hook", nil)
|
||||
n.Notify(NodeLost, "n1", "node n1 was lost", nil)
|
||||
n.Notify(NodeLost, "n1", "node n1 was lost again", nil) // within the hour: not sent
|
||||
n.Notify(NodeLost, "n2", "node n2 was lost", nil)
|
||||
n.Wait()
|
||||
|
||||
if len(g.calls) != 2 {
|
||||
t.Fatalf("%d calls, want 2: %q", len(g.calls), g.body)
|
||||
}
|
||||
var m map[string]string
|
||||
json.Unmarshal(g.body[0], &m)
|
||||
if !strings.Contains(m["text"], "gw1") || !strings.Contains(m["text"], "node n1 was lost") {
|
||||
t.Errorf("text %q", m["text"])
|
||||
}
|
||||
if g.calls[0].Header.Get("X-Llmgw-Signature") != "" {
|
||||
t.Error("signed without a secret")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetries(t *testing.T) {
|
||||
var g got
|
||||
srv := httptest.NewServer(g.handler(2))
|
||||
defer srv.Close()
|
||||
n := newNotifier(t, Config{Webhooks: []Webhook{{URL: srv.URL}}})
|
||||
n.Notify(BudgetSpent, "org:1", "spent", nil)
|
||||
n.Wait()
|
||||
if len(g.calls) != 3 {
|
||||
t.Fatalf("%d attempts, want 3 (two failures, then delivered)", len(g.calls))
|
||||
}
|
||||
|
||||
var h got
|
||||
srv2 := httptest.NewServer(h.handler(10))
|
||||
defer srv2.Close()
|
||||
n = newNotifier(t, Config{Webhooks: []Webhook{{URL: srv2.URL}}})
|
||||
n.Notify(BudgetSpent, "org:1", "spent", nil)
|
||||
n.Wait()
|
||||
if len(h.calls) != 3 {
|
||||
t.Fatalf("%d attempts at a failing hook, want 3 then given up", len(h.calls))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNilAndValidate(t *testing.T) {
|
||||
var n *Notifier
|
||||
n.Notify(NodeLost, "x", "y", nil) // must not panic
|
||||
n.Wait()
|
||||
if n, err := New(context.Background(), Config{}, nil); n != nil || err != nil {
|
||||
t.Fatalf("nothing configured: %v %v", n, err)
|
||||
}
|
||||
for _, c := range []Config{
|
||||
{Webhooks: []Webhook{{URL: "ftp://x"}}},
|
||||
{Webhooks: []Webhook{{URL: "https://x", Format: "teams"}}},
|
||||
{Webhooks: []Webhook{{URL: "https://x", Events: []string{"model_gone"}}}},
|
||||
} {
|
||||
if c.Validate() == nil {
|
||||
t.Errorf("%+v accepted", c)
|
||||
}
|
||||
}
|
||||
if _, err := New(context.Background(), Config{Webhooks: []Webhook{{URL: "https://x", SecretFile: "/nonexistent"}}}, nil); err == nil {
|
||||
t.Error("a missing secret file accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTestSaysHowEachAnsweredWithoutTheSecretPath(t *testing.T) {
|
||||
var g got
|
||||
ok := httptest.NewServer(g.handler(0))
|
||||
defer ok.Close()
|
||||
bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusForbidden) }))
|
||||
defer bad.Close()
|
||||
n := newNotifier(t, Config{Webhooks: []Webhook{
|
||||
{URL: ok.URL + "/services/T0/B0/s3cret"},
|
||||
{URL: bad.URL + "/hook"},
|
||||
{URL: "http://127.0.0.1:1/services/T0/B0/s3cret"},
|
||||
}})
|
||||
defer n.Wait()
|
||||
res := n.Test(context.Background())
|
||||
if len(res) != 3 || res[0].Err != "" || !strings.Contains(res[1].Err, "403") || res[2].Err == "" {
|
||||
t.Fatalf("%+v", res)
|
||||
}
|
||||
for _, r := range res {
|
||||
if strings.Contains(r.URL+r.Err, "s3cret") {
|
||||
t.Errorf("the secret path shown: %+v", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
// Package notifytest catches notifications in tests.
|
||||
package notifytest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
)
|
||||
|
||||
// Catch returns a notifier whose webhook records what it is sent, and a func
|
||||
// that waits for the sending and returns the events. The notifier takes no
|
||||
// more events once that func is called.
|
||||
func Catch(t *testing.T) (*notify.Notifier, func() []notify.Event) {
|
||||
t.Helper()
|
||||
var mu sync.Mutex
|
||||
var got []notify.Event
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
b, _ := io.ReadAll(req.Body)
|
||||
var e notify.Event
|
||||
_ = json.Unmarshal(b, &e)
|
||||
mu.Lock()
|
||||
got = append(got, e)
|
||||
mu.Unlock()
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
n, err := notify.New(t.Context(), notify.Config{Webhooks: []notify.Webhook{{URL: srv.URL}}}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n, func() []notify.Event {
|
||||
n.Wait()
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return got
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user