// Package notify tells the people running the gateway about what needs them, // without a Prometheus in between: a model parked after failing every start, // a node lost, a budget spent, a model's repository with new weights, a // guardrails judge not answering. Each // event goes to the webhooks configured for it — JSON signed with a shared // secret, or a Slack-compatible message. // // Sending never holds up what raised the event: events are queued, and a // webhook that does not answer is retried a few times and then given up on. // The same event about the same thing is sent once an hour at most. package notify import ( "bytes" "context" "crypto/hmac" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "log/slog" "net/http" "net/url" "os" "slices" "strings" "sync" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/secrets" ) // Event kinds. const ( ModelParked = "model_parked" NodeLost = "node_lost" BudgetSpent = "budget_spent" UpdateAvailable = "update_available" // JudgeUnavailable is a guardrails judge giving no verdict: with // fail: closed, requests are being refused; with open, served unjudged. JudgeUnavailable = "judge_unavailable" // CanaryRolledBack is a model's canary taken out because it served // worse than the model it was a candidate to replace. CanaryRolledBack = "canary_rolled_back" ) // Kinds are every event there is. var Kinds = []string{ModelParked, NodeLost, BudgetSpent, UpdateAvailable, JudgeUnavailable, CanaryRolledBack} // Event is one thing that happened. type Event struct { Kind string `json:"kind"` At time.Time `json:"at"` Subject string `json:"subject"` Message string `json:"message"` Detail map[string]any `json:"detail,omitempty"` } // Webhook is where events go. type Webhook struct { URL string `koanf:"url"` // Format is json (the event, with X-Llmgw-Signature when SecretFile is // set) or slack ({"text": …}, which Slack, Mattermost and Rocket.Chat // incoming webhooks take). Format string `koanf:"format"` // Events are the kinds sent here; empty is all. Events []string `koanf:"events"` // SecretFile holds the key the body is signed with: X-Llmgw-Signature // is sha256=. SecretFile string `koanf:"secret_file"` } // Config is server.yaml's notify: section. type Config struct { Webhooks []Webhook `koanf:"webhooks"` // Host names the gateway in messages; empty is this machine's name. Host string `koanf:"host"` } // Validate checks the settings. func (c Config) Validate() error { for i, w := range c.Webhooks { if !strings.HasPrefix(w.URL, "https://") && !strings.HasPrefix(w.URL, "http://") { return fmt.Errorf("notify.webhooks[%d].url %q: an http(s) address", i, w.URL) } switch w.Format { case "", "json", "slack": default: return fmt.Errorf("notify.webhooks[%d].format %q: json or slack", i, w.Format) } for _, e := range w.Events { if !slices.Contains(Kinds, e) { return fmt.Errorf("notify.webhooks[%d].events: %q is not one of %s", i, e, strings.Join(Kinds, ", ")) } } } return nil } // Notifier sends events. A nil one sends nothing. type Notifier struct { hooks []hook host string log *slog.Logger client *http.Client queue chan Event retry time.Duration mu sync.Mutex sent map[string]time.Time wg sync.WaitGroup claim Claimer } // Claimer is where control planes agree which of them sends an event: each // raises the same ones — a budget is spent on every control plane serving // its requests — and only the one that claims an event sends it. type Claimer interface { ClaimNotice(ctx context.Context, key string, at time.Time, window time.Duration) (bool, error) } // SetClaimer has events claimed before they are sent. Set before the first // event; nil leaves each process to send its own. func (n *Notifier) SetClaimer(c Claimer) { if n != nil { n.claim = c } } type hook struct { Webhook secret []byte } // dedupe is how long the same event about the same thing is not sent again. const dedupe = time.Hour // New builds a notifier and starts sending; it stops when ctx ends. Nil when // nothing is configured. func New(ctx context.Context, c Config, log *slog.Logger) (*Notifier, error) { if len(c.Webhooks) == 0 { return nil, nil } if err := c.Validate(); err != nil { return nil, err } if log == nil { log = slog.Default() } n := &Notifier{host: c.Host, log: log, client: &http.Client{Timeout: 10 * time.Second}, queue: make(chan Event, 256), sent: map[string]time.Time{}, retry: 5 * time.Second} if n.host == "" { n.host, _ = os.Hostname() } for _, w := range c.Webhooks { h := hook{Webhook: w} if w.SecretFile != "" { b, err := secrets.Read(ctx, w.SecretFile) if err != nil { return nil, fmt.Errorf("notify: %w", err) } h.secret = bytes.TrimSpace(b) } n.hooks = append(n.hooks, h) } n.wg.Add(1) go n.run(ctx) return n, nil } // Notify queues an event. It never blocks: with the queue full the event is // dropped, and said so in the log. func (n *Notifier) Notify(kind, subject, message string, detail map[string]any) { if n == nil { return } k := kind + "\x00" + subject now := time.Now() n.mu.Lock() if last, ok := n.sent[k]; ok && now.Sub(last) < dedupe { n.mu.Unlock() return } n.sent[k] = now for key, at := range n.sent { if now.Sub(at) > dedupe { delete(n.sent, key) } } n.mu.Unlock() select { case n.queue <- Event{Kind: kind, At: now.UTC(), Subject: subject, Message: message, Detail: detail}: default: n.log.Warn("a notification was dropped: too many waiting", "kind", kind, "subject", subject) } } // Wait waits for what is queued to be sent, for tests and shutdown. func (n *Notifier) Wait() { if n == nil { return } close(n.queue) n.wg.Wait() } func (n *Notifier) run(ctx context.Context) { defer n.wg.Done() for { select { case <-ctx.Done(): return case e, ok := <-n.queue: if !ok { return } if !n.claimed(ctx, e) { continue } for _, h := range n.hooks { if len(h.Events) > 0 && !slices.Contains(h.Events, e.Kind) { continue } if err := n.send(ctx, h, e, 3); err != nil { n.log.Warn("a notification could not be delivered", "kind", e.Kind, "subject", e.Subject, "err", err) } } } } } // Test sends a test event to every webhook now, once each, and says how each // answered: what `llmgw ctl notify test` shows. func (n *Notifier) Test(ctx context.Context) []Result { if n == nil { return nil } e := Event{Kind: "test", At: time.Now().UTC(), Subject: "test", Message: "a test notification: this webhook is reached"} var out []Result for _, h := range n.hooks { r := Result{URL: redact(h.URL)} if err := n.send(ctx, h, e, 1); err != nil { r.Err = err.Error() } out = append(out, r) } return out } // Result is how one webhook answered a test. type Result struct { URL string `json:"url"` Err string `json:"error,omitempty"` } // redact keeps a webhook's host and drops its path, where Slack and others // put the secret. func redact(u string) string { if i := strings.Index(u, "://"); i >= 0 { if j := strings.IndexByte(u[i+3:], '/'); j >= 0 { return u[:i+3+j] + "/…" } } return u } func (n *Notifier) send(ctx context.Context, h hook, e Event, attempts int) error { var body []byte if h.Format == "slack" { body, _ = json.Marshal(map[string]string{"text": fmt.Sprintf("*llmgw on %s* — %s", n.host, e.Message)}) } else { ev := e if ev.Detail == nil { ev.Detail = map[string]any{} } ev.Detail["gateway"] = n.host body, _ = json.Marshal(ev) } for attempt := 1; ; attempt++ { req, err := http.NewRequestWithContext(ctx, http.MethodPost, h.URL, bytes.NewReader(body)) if err != nil { return err } req.Header.Set("Content-Type", "application/json") req.Header.Set("X-Llmgw-Event", e.Kind) if h.secret != nil { m := hmac.New(sha256.New, h.secret) m.Write(body) req.Header.Set("X-Llmgw-Signature", "sha256="+hex.EncodeToString(m.Sum(nil))) } resp, err := n.client.Do(req) // Not the error as it comes: it names the whole address, and the // path of a Slack or PagerDuty webhook is its secret. var ue *url.Error if errors.As(err, &ue) { err = fmt.Errorf("%s %s: %w", ue.Op, redact(h.URL), ue.Err) } if err == nil { resp.Body.Close() if resp.StatusCode < 300 { return nil } err = fmt.Errorf("answered %s", resp.Status) } if attempt >= attempts { return err } select { case <-ctx.Done(): return ctx.Err() case <-time.After(n.retry * time.Duration(attempt)): } } } // claimed reports whether this process is the one to send an event. A store // that cannot be asked sends it anyway: told twice beats not told. func (n *Notifier) claimed(ctx context.Context, e Event) bool { if n.claim == nil { return true } cctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() ok, err := n.claim.ClaimNotice(cctx, e.Kind+"\x00"+e.Subject, e.At, dedupe) if err != nil { n.log.Warn("could not ask the store whether a notification was sent; sending it", "kind", e.Kind, "err", err) return true } return ok }