llmgw/internal/notify/notify.go

// Package notify tells the people running the gateway about what needs them,
// without a Prometheus in between: a model parked after failing every start,
// a node lost, a budget spent, a model's repository with new weights, a
// guardrails judge not answering. Each
// event goes to the webhooks configured for it — JSON signed with a shared
// secret, or a Slack-compatible message.
//
// Sending never holds up what raised the event: events are queued, and a
// webhook that does not answer is retried a few times and then given up on.
// The same event about the same thing is sent once an hour at most.
package notify

import (
	"bytes"
	"context"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"log/slog"
	"net/http"
	"net/url"
	"os"
	"slices"
	"strings"
	"sync"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/secrets"
)

// Event kinds.
const (
	ModelParked     = "model_parked"
	NodeLost        = "node_lost"
	BudgetSpent     = "budget_spent"
	UpdateAvailable = "update_available"
	// JudgeUnavailable is a guardrails judge giving no verdict: with
	// fail: closed, requests are being refused; with open, served unjudged.
	JudgeUnavailable = "judge_unavailable"
	// CanaryRolledBack is a model's canary taken out because it served
	// worse than the model it was a candidate to replace.
	CanaryRolledBack = "canary_rolled_back"
)

// Kinds are every event there is.
var Kinds = []string{ModelParked, NodeLost, BudgetSpent, UpdateAvailable, JudgeUnavailable, CanaryRolledBack}

// Event is one thing that happened.
type Event struct {
	Kind    string         `json:"kind"`
	At      time.Time      `json:"at"`
	Subject string         `json:"subject"`
	Message string         `json:"message"`
	Detail  map[string]any `json:"detail,omitempty"`
}

// Webhook is where events go.
type Webhook struct {
	URL string `koanf:"url"`
	// Format is json (the event, with X-Llmgw-Signature when SecretFile is
	// set) or slack ({"text": …}, which Slack, Mattermost and Rocket.Chat
	// incoming webhooks take).
	Format string `koanf:"format"`
	// Events are the kinds sent here; empty is all.
	Events []string `koanf:"events"`
	// SecretFile holds the key the body is signed with: X-Llmgw-Signature
	// is sha256=<hex HMAC of the body>.
	SecretFile string `koanf:"secret_file"`
}

// Config is server.yaml's notify: section.
type Config struct {
	Webhooks []Webhook `koanf:"webhooks"`
	// Host names the gateway in messages; empty is this machine's name.
	Host string `koanf:"host"`
}

// Validate checks the settings.
func (c Config) Validate() error {
	for i, w := range c.Webhooks {
		if !strings.HasPrefix(w.URL, "https://") && !strings.HasPrefix(w.URL, "http://") {
			return fmt.Errorf("notify.webhooks[%d].url %q: an http(s) address", i, w.URL)
		}
		switch w.Format {
		case "", "json", "slack":
		default:
			return fmt.Errorf("notify.webhooks[%d].format %q: json or slack", i, w.Format)
		}
		for _, e := range w.Events {
			if !slices.Contains(Kinds, e) {
				return fmt.Errorf("notify.webhooks[%d].events: %q is not one of %s", i, e, strings.Join(Kinds, ", "))
			}
		}
	}
	return nil
}

// Notifier sends events. A nil one sends nothing.
type Notifier struct {
	hooks  []hook
	host   string
	log    *slog.Logger
	client *http.Client
	queue  chan Event
	retry  time.Duration

	mu   sync.Mutex
	sent map[string]time.Time
	wg   sync.WaitGroup

	claim Claimer
}

// Claimer is where control planes agree which of them sends an event: each
// raises the same ones — a budget is spent on every control plane serving
// its requests — and only the one that claims an event sends it.
type Claimer interface {
	ClaimNotice(ctx context.Context, key string, at time.Time, window time.Duration) (bool, error)
}

// SetClaimer has events claimed before they are sent. Set before the first
// event; nil leaves each process to send its own.
func (n *Notifier) SetClaimer(c Claimer) {
	if n != nil {
		n.claim = c
	}
}

type hook struct {
	Webhook
	secret []byte
}

// dedupe is how long the same event about the same thing is not sent again.
const dedupe = time.Hour

// New builds a notifier and starts sending; it stops when ctx ends. Nil when
// nothing is configured.
func New(ctx context.Context, c Config, log *slog.Logger) (*Notifier, error) {
	if len(c.Webhooks) == 0 {
		return nil, nil
	}
	if err := c.Validate(); err != nil {
		return nil, err
	}
	if log == nil {
		log = slog.Default()
	}
	n := &Notifier{host: c.Host, log: log, client: &http.Client{Timeout: 10 * time.Second},
		queue: make(chan Event, 256), sent: map[string]time.Time{}, retry: 5 * time.Second}
	if n.host == "" {
		n.host, _ = os.Hostname()
	}
	for _, w := range c.Webhooks {
		h := hook{Webhook: w}
		if w.SecretFile != "" {
			b, err := secrets.Read(ctx, w.SecretFile)
			if err != nil {
				return nil, fmt.Errorf("notify: %w", err)
			}
			h.secret = bytes.TrimSpace(b)
		}
		n.hooks = append(n.hooks, h)
	}
	n.wg.Add(1)
	go n.run(ctx)
	return n, nil
}

// Notify queues an event. It never blocks: with the queue full the event is
// dropped, and said so in the log.
func (n *Notifier) Notify(kind, subject, message string, detail map[string]any) {
	if n == nil {
		return
	}
	k := kind + "\x00" + subject
	now := time.Now()
	n.mu.Lock()
	if last, ok := n.sent[k]; ok && now.Sub(last) < dedupe {
		n.mu.Unlock()
		return
	}
	n.sent[k] = now
	for key, at := range n.sent {
		if now.Sub(at) > dedupe {
			delete(n.sent, key)
		}
	}
	n.mu.Unlock()
	select {
	case n.queue <- Event{Kind: kind, At: now.UTC(), Subject: subject, Message: message, Detail: detail}:
	default:
		n.log.Warn("a notification was dropped: too many waiting", "kind", kind, "subject", subject)
	}
}

// Wait waits for what is queued to be sent, for tests and shutdown.
func (n *Notifier) Wait() {
	if n == nil {
		return
	}
	close(n.queue)
	n.wg.Wait()
}

func (n *Notifier) run(ctx context.Context) {
	defer n.wg.Done()
	for {
		select {
		case <-ctx.Done():
			return
		case e, ok := <-n.queue:
			if !ok {
				return
			}
			if !n.claimed(ctx, e) {
				continue
			}
			for _, h := range n.hooks {
				if len(h.Events) > 0 && !slices.Contains(h.Events, e.Kind) {
					continue
				}
				if err := n.send(ctx, h, e, 3); err != nil {
					n.log.Warn("a notification could not be delivered", "kind", e.Kind, "subject", e.Subject, "err", err)
				}
			}
		}
	}
}

// Test sends a test event to every webhook now, once each, and says how each
// answered: what `llmgw ctl notify test` shows.
func (n *Notifier) Test(ctx context.Context) []Result {
	if n == nil {
		return nil
	}
	e := Event{Kind: "test", At: time.Now().UTC(), Subject: "test",
		Message: "a test notification: this webhook is reached"}
	var out []Result
	for _, h := range n.hooks {
		r := Result{URL: redact(h.URL)}
		if err := n.send(ctx, h, e, 1); err != nil {
			r.Err = err.Error()
		}
		out = append(out, r)
	}
	return out
}

// Result is how one webhook answered a test.
type Result struct {
	URL string `json:"url"`
	Err string `json:"error,omitempty"`
}

// redact keeps a webhook's host and drops its path, where Slack and others
// put the secret.
func redact(u string) string { return Redact(u) }

// Redact is a webhook's address as it may be shown: its host, not the path
// that is its secret.
func Redact(u string) string {
	if i := strings.Index(u, "://"); i >= 0 {
		if j := strings.IndexByte(u[i+3:], '/'); j >= 0 {
			return u[:i+3+j] + "/…"
		}
	}
	return u
}

func (n *Notifier) send(ctx context.Context, h hook, e Event, attempts int) error {
	var body []byte
	if h.Format == "slack" {
		body, _ = json.Marshal(map[string]string{"text": fmt.Sprintf("*llmgw on %s* — %s", n.host, e.Message)})
	} else {
		ev := e
		if ev.Detail == nil {
			ev.Detail = map[string]any{}
		}
		ev.Detail["gateway"] = n.host
		body, _ = json.Marshal(ev)
	}
	for attempt := 1; ; attempt++ {
		req, err := http.NewRequestWithContext(ctx, http.MethodPost, h.URL, bytes.NewReader(body))
		if err != nil {
			return err
		}
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("X-Llmgw-Event", e.Kind)
		if h.secret != nil {
			m := hmac.New(sha256.New, h.secret)
			m.Write(body)
			req.Header.Set("X-Llmgw-Signature", "sha256="+hex.EncodeToString(m.Sum(nil)))
		}
		resp, err := n.client.Do(req)
		// Not the error as it comes: it names the whole address, and the
		// path of a Slack or PagerDuty webhook is its secret.
		var ue *url.Error
		if errors.As(err, &ue) {
			err = fmt.Errorf("%s %s: %w", ue.Op, redact(h.URL), ue.Err)
		}
		if err == nil {
			resp.Body.Close()
			if resp.StatusCode < 300 {
				return nil
			}
			err = fmt.Errorf("answered %s", resp.Status)
		}
		if attempt >= attempts {
			return err
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(n.retry * time.Duration(attempt)):
		}
	}
}

// claimed reports whether this process is the one to send an event. A store
// that cannot be asked sends it anyway: told twice beats not told.
func (n *Notifier) claimed(ctx context.Context, e Event) bool {
	if n.claim == nil {
		return true
	}
	cctx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()
	ok, err := n.claim.ClaimNotice(cctx, e.Kind+"\x00"+e.Subject, e.At, dedupe)
	if err != nil {
		n.log.Warn("could not ask the store whether a notification was sent; sending it", "kind", e.Kind, "err", err)
		return true
	}
	return ok
}