Notifications: webhooks for a parked model, a lost node, a spent budget and new weights; Alertmanager receiver templates

notify.webhooks sends signed JSON or Slack-compatible messages, queued,
retried, de-duplicated hourly; ctl notify test checks each webhook.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 16:58:11 +02:00
1 parent 646ff996c7
commit 1b71c38f89
24 files changed
+907 -4

No files matched your search

+4
View File
@@ -35,11 +35,14 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// API serves the management endpoints.
type API struct {
// notifier is the webhooks `ctl notify test` reaches; nil without any.
notifier *notify.Notifier
// releases is where release binaries are fetched from, for upgrades.
releases string
// storeStatus reads a replicated store's members; nil for one database.
@@ -84,6 +87,7 @@ func (a *API) Register(mux *http.ServeMux) {
a.registerGuard(mux)
mux.HandleFunc("GET /api/v1/audit", a.auditEvents)
mux.HandleFunc("GET /api/v1/usage/export", a.usageExport)
mux.HandleFunc("POST /api/v1/notify/test", a.notifyTest)
mux.HandleFunc("POST /api/v1/models/{name}/adapters", a.setAdapters)
mux.HandleFunc("GET /api/v1/models/{name}/target", a.target)
mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) {
+19
View File
@@ -0,0 +1,19 @@
package adminapi
import (
"net/http"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
)
// SetNotifier gives the API the webhooks `ctl notify test` reaches.
func (a *API) SetNotifier(n *notify.Notifier) { a.notifier = n }
// notifyTest sends a test event to every webhook and says how each answered.
func (a *API) notifyTest(w http.ResponseWriter, r *http.Request) {
if a.notifier == nil {
writeError(w, http.StatusNotFound, "no webhooks: server.yaml has no notify.webhooks")
return
}
writeJSON(w, http.StatusOK, a.notifier.Test(r.Context()))
}
@@ -31,6 +31,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
)
// Registry reads a repository.
@@ -85,6 +86,9 @@ type Deps struct {
// UnplacedGrace is how long a model may go unplaced before the job
// stops waiting for it: a minute unless set.
UnplacedGrace time.Duration
// Notify is told when a model's repository publishes new weights; nil
// tells no one.
Notify *notify.Notifier
}
// errBusy is a model the cluster has the hardware for, all of it in use now.
+9
View File
@@ -16,6 +16,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"github.com/prometheus/client_golang/prometheus"
)
@@ -343,8 +344,16 @@ func (s *Server) checkOne(ctx context.Context, m catalog.Model, o *catalog.Origi
}
}
s.mu.Lock()
prev, had := s.updates[m.Name]
s.updates[m.Name] = u
s.mu.Unlock()
// Told once per new revision, not at every check that finds it still there.
if u.Available && (!had || !prev.Available || prev.Latest != u.Latest) {
s.deps.Notify.Notify(notify.UpdateAvailable, m.Name+"@"+u.Latest,
fmt.Sprintf("%s has new weights at %s (%s); `llmgw ctl model update %s` moves to them",
m.Name, o.Repo, catalog.ShortRevision(u.Latest), m.Name),
map[string]any{"model": m.Name, "repo": o.Repo, "current": o.Revision, "latest": u.Latest})
}
return u
}
@@ -14,6 +14,8 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify/notifytest"
)
const (
@@ -338,3 +340,15 @@ func TestAnOriginGivenByHandIsChecked(t *testing.T) {
t.Fatalf("%+v", o)
}
}
func TestNewWeightsAreToldOncePerRevision(t *testing.T) {
s, _, _, _, _ := updSetup(t)
n, sent := notifytest.Catch(t)
s.deps.Notify = n
s.CheckUpdates(context.Background())
s.CheckUpdates(context.Background()) // still there: not told again
got := sent()
if len(got) != 1 || got[0].Kind != notify.UpdateAvailable || got[0].Detail["model"] != "qwen" {
t.Fatalf("sent %+v, want one update_available for qwen", got)
}
}
@@ -0,0 +1,46 @@
package reconciler
import (
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify/notifytest"
)
func TestParkingAModelTellsSomeone(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
n, sent := notifytest.Catch(t)
h.r.SetNotifier(n)
failRepeatedly(t, h, DefaultConfig().ParkAfter, "unknown model architecture: 'ggmlc'")
h.advance(DefaultConfig().MaxBackoff + time.Second)
h.r.Reconcile(t.Context())
got := sent()
if len(got) != 1 || got[0].Kind != notify.ModelParked || got[0].Subject != "llama" {
t.Fatalf("sent %+v, want one model_parked for llama", got)
}
}
func TestLosingANodeTellsSomeone(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
n, sent := notifytest.Catch(t)
h.r.SetNotifier(n)
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
for _, d := range []time.Duration{20 * time.Second, 90 * time.Second} {
h.clock.advance(d)
touchAt(h.inv, "2", h.clock.now())
h.r.Reconcile(t.Context())
}
if n, _ := h.inv.Get("1"); n.State != inventory.Lost {
t.Fatalf("node 1 is %s", n.State)
}
got := sent()
if len(got) != 1 || got[0].Kind != notify.NodeLost || got[0].Subject != "1" {
t.Fatalf("sent %+v, want one node_lost for 1", got)
}
}
+18
View File
@@ -8,6 +8,7 @@ import (
"math/rand/v2"
"slices"
"sync"
"sync/atomic"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
@@ -16,6 +17,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
)
// ShardCommand is everything an agent needs to launch one shard.
@@ -231,6 +233,9 @@ type Reconciler struct {
disp Dispatcher
log *slog.Logger
now func() time.Time
// notify tells people about parked models and lost nodes; nil tells
// no one.
notify atomic.Pointer[notify.Notifier]
mu sync.Mutex
replicas map[string]*Replica
@@ -559,6 +564,9 @@ func (r *Reconciler) expireNodes() {
r.inv.SetState(n.ID, inventory.Lost)
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
"node", n.ID, "silent_for", silent.Round(time.Second))
r.notify.Load().Notify(notify.NodeLost, n.ID,
fmt.Sprintf("node %s was lost: silent for %s; its replicas are rebuilt elsewhere", n.ID, silent.Round(time.Second)),
map[string]any{"node": n.ID, "silent_seconds": int(silent.Seconds())})
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")
}
}
@@ -644,6 +652,10 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
r.log.Error("model parked: it failed every time it started, and is not retried "+
"until its definition changes or `llmgw ctl retry` is run",
"model", rep.Model, "failures", n, "last_error", rep.Err)
r.notify.Load().Notify(notify.ModelParked, rep.Model,
fmt.Sprintf("model %s is parked: it failed %d times in a row (%s); `llmgw ctl retry %s` tries again",
rep.Model, n, rep.Err, rep.Model),
map[string]any{"model": rep.Model, "failures": n, "last_error": rep.Err})
}
}
}
@@ -1216,3 +1228,9 @@ func (r *Reconciler) engineKey(replicaID string) string {
}
return r.cfg.EngineKey(replicaID)
}
// SetNotifier gives the reconciler somewhere to say a model was parked or a
// node lost.
func (r *Reconciler) SetNotifier(n *notify.Notifier) {
r.notify.Store(n)
}