Notifications: webhooks for a parked model, a lost node, a spent budget and new weights; Alertmanager receiver templates

notify.webhooks sends signed JSON or Slack-compatible messages, queued,
retried, de-duplicated hourly; ctl notify test checks each webhook.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 16:58:11 +02:00
1 parent 646ff996c7
commit 1b71c38f89
24 files changed
+907 -4

No files matched your search

+11
View File
@@ -6,9 +6,11 @@ import (
"fmt"
"log/slog"
"strings"
"sync/atomic"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
@@ -45,8 +47,13 @@ type Accountant struct {
tokens *Limiter
recorder *Recorder
log *slog.Logger
// notify is told when a budget runs out; nil tells no one.
notify atomic.Pointer[notify.Notifier]
}
// SetNotifier gives the accountant somewhere to say a budget is spent.
func (a *Accountant) SetNotifier(n *notify.Notifier) { a.notify.Store(n) }
// New builds an accountant.
func New(pricing *Pricing, recorder *Recorder, log *slog.Logger) *Accountant {
if log == nil {
@@ -182,6 +189,10 @@ func (a *Accountant) Check(p auth.Principal, model string, estimatedTokens int64
if !price.Zero() {
for _, l := range chain {
if l.Budget != nil && a.budgets.Exhausted(l.Counter, l.Period, *l.Budget) {
a.notify.Load().Notify(notify.BudgetSpent, l.Counter,
fmt.Sprintf("the %s budget of %s is spent: requests to priced models are refused until it renews",
or(string(l.Period), "total"), l.Counter),
map[string]any{"counter": l.Counter, "period": string(l.Period), "budget": *l.Budget})
return res, &LimitError{Err: ErrBudgetSpent}
}
}
+27
View File
@@ -5,11 +5,14 @@ import (
"errors"
"log/slog"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify/notifytest"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
@@ -677,3 +680,27 @@ func TestAFallbackIsChargedWhatTheModelThatAnsweredCosts(t *testing.T) {
t.Error("the paid model answering cost nothing")
}
}
func TestASpentBudgetTellsSomeoneOnce(t *testing.T) {
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
n, sent := notifytest.Catch(t)
a.SetNotifier(n)
p := principal("k1", func(e *store.Effective) {
e.Budget = i64p(100)
e.BudgetPeriod = store.PeriodMonth
})
res, err := a.Check(p, "paid", 10)
if err != nil {
t.Fatal(err)
}
a.Record(res, store.Usage{KeyID: "k1", PromptTokens: 1000})
for range 3 {
if _, err := a.Check(p, "paid", 10); !errors.Is(err, ErrBudgetSpent) {
t.Fatalf("not refused: %v", err)
}
}
got := sent()
if len(got) != 1 || got[0].Kind != notify.BudgetSpent || !strings.Contains(got[0].Message, "month") {
t.Fatalf("sent %+v, want one budget_spent", got)
}
}