Notifications: webhooks for a parked model, a lost node, a spent budget and new weights; Alertmanager receiver templates

notify.webhooks sends signed JSON or Slack-compatible messages, queued,
retried, de-duplicated hourly; ctl notify test checks each webhook.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 16:58:11 +02:00
1 parent 646ff996c7
commit 1b71c38f89
24 files changed
+907 -4

No files matched your search

@@ -0,0 +1,46 @@
package reconciler
import (
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify/notifytest"
)
func TestParkingAModelTellsSomeone(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
n, sent := notifytest.Catch(t)
h.r.SetNotifier(n)
failRepeatedly(t, h, DefaultConfig().ParkAfter, "unknown model architecture: 'ggmlc'")
h.advance(DefaultConfig().MaxBackoff + time.Second)
h.r.Reconcile(t.Context())
got := sent()
if len(got) != 1 || got[0].Kind != notify.ModelParked || got[0].Subject != "llama" {
t.Fatalf("sent %+v, want one model_parked for llama", got)
}
}
func TestLosingANodeTellsSomeone(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
n, sent := notifytest.Catch(t)
h.r.SetNotifier(n)
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
for _, d := range []time.Duration{20 * time.Second, 90 * time.Second} {
h.clock.advance(d)
touchAt(h.inv, "2", h.clock.now())
h.r.Reconcile(t.Context())
}
if n, _ := h.inv.Get("1"); n.State != inventory.Lost {
t.Fatalf("node 1 is %s", n.State)
}
got := sent()
if len(got) != 1 || got[0].Kind != notify.NodeLost || got[0].Subject != "1" {
t.Fatalf("sent %+v, want one node_lost for 1", got)
}
}
+18
View File
@@ -8,6 +8,7 @@ import (
"math/rand/v2"
"slices"
"sync"
"sync/atomic"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
@@ -16,6 +17,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
)
// ShardCommand is everything an agent needs to launch one shard.
@@ -231,6 +233,9 @@ type Reconciler struct {
disp Dispatcher
log *slog.Logger
now func() time.Time
// notify tells people about parked models and lost nodes; nil tells
// no one.
notify atomic.Pointer[notify.Notifier]
mu sync.Mutex
replicas map[string]*Replica
@@ -559,6 +564,9 @@ func (r *Reconciler) expireNodes() {
r.inv.SetState(n.ID, inventory.Lost)
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
"node", n.ID, "silent_for", silent.Round(time.Second))
r.notify.Load().Notify(notify.NodeLost, n.ID,
fmt.Sprintf("node %s was lost: silent for %s; its replicas are rebuilt elsewhere", n.ID, silent.Round(time.Second)),
map[string]any{"node": n.ID, "silent_seconds": int(silent.Seconds())})
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")
}
}
@@ -644,6 +652,10 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
r.log.Error("model parked: it failed every time it started, and is not retried "+
"until its definition changes or `llmgw ctl retry` is run",
"model", rep.Model, "failures", n, "last_error", rep.Err)
r.notify.Load().Notify(notify.ModelParked, rep.Model,
fmt.Sprintf("model %s is parked: it failed %d times in a row (%s); `llmgw ctl retry %s` tries again",
rep.Model, n, rep.Err, rep.Model),
map[string]any{"model": rep.Model, "failures": n, "last_error": rep.Err})
}
}
}
@@ -1216,3 +1228,9 @@ func (r *Reconciler) engineKey(replicaID string) string {
}
return r.cfg.EngineKey(replicaID)
}
// SetNotifier gives the reconciler somewhere to say a model was parked or a
// node lost.
func (r *Reconciler) SetNotifier(n *notify.Notifier) {
r.notify.Store(n)
}