Notifications: webhooks for a parked model, a lost node, a spent budget and new weights; Alertmanager receiver templates
notify.webhooks sends signed JSON or Slack-compatible messages, queued, retried, de-duplicated hourly; ctl notify test checks each webhook. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
646ff996c7
commit
1b71c38f89
24 files changed
+907
-4
No files matched your search
@@ -0,0 +1,46 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify/notifytest"
|
||||
)
|
||||
|
||||
func TestParkingAModelTellsSomeone(t *testing.T) {
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
n, sent := notifytest.Catch(t)
|
||||
h.r.SetNotifier(n)
|
||||
failRepeatedly(t, h, DefaultConfig().ParkAfter, "unknown model architecture: 'ggmlc'")
|
||||
h.advance(DefaultConfig().MaxBackoff + time.Second)
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
got := sent()
|
||||
if len(got) != 1 || got[0].Kind != notify.ModelParked || got[0].Subject != "llama" {
|
||||
t.Fatalf("sent %+v, want one model_parked for llama", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLosingANodeTellsSomeone(t *testing.T) {
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
|
||||
n, sent := notifytest.Catch(t)
|
||||
h.r.SetNotifier(n)
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
for _, d := range []time.Duration{20 * time.Second, 90 * time.Second} {
|
||||
h.clock.advance(d)
|
||||
touchAt(h.inv, "2", h.clock.now())
|
||||
h.r.Reconcile(t.Context())
|
||||
}
|
||||
if n, _ := h.inv.Get("1"); n.State != inventory.Lost {
|
||||
t.Fatalf("node 1 is %s", n.State)
|
||||
}
|
||||
got := sent()
|
||||
if len(got) != 1 || got[0].Kind != notify.NodeLost || got[0].Subject != "1" {
|
||||
t.Fatalf("sent %+v, want one node_lost for 1", got)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"math/rand/v2"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
@@ -16,6 +17,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
)
|
||||
|
||||
// ShardCommand is everything an agent needs to launch one shard.
|
||||
@@ -231,6 +233,9 @@ type Reconciler struct {
|
||||
disp Dispatcher
|
||||
log *slog.Logger
|
||||
now func() time.Time
|
||||
// notify tells people about parked models and lost nodes; nil tells
|
||||
// no one.
|
||||
notify atomic.Pointer[notify.Notifier]
|
||||
|
||||
mu sync.Mutex
|
||||
replicas map[string]*Replica
|
||||
@@ -559,6 +564,9 @@ func (r *Reconciler) expireNodes() {
|
||||
r.inv.SetState(n.ID, inventory.Lost)
|
||||
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
|
||||
"node", n.ID, "silent_for", silent.Round(time.Second))
|
||||
r.notify.Load().Notify(notify.NodeLost, n.ID,
|
||||
fmt.Sprintf("node %s was lost: silent for %s; its replicas are rebuilt elsewhere", n.ID, silent.Round(time.Second)),
|
||||
map[string]any{"node": n.ID, "silent_seconds": int(silent.Seconds())})
|
||||
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")
|
||||
}
|
||||
}
|
||||
@@ -644,6 +652,10 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
|
||||
r.log.Error("model parked: it failed every time it started, and is not retried "+
|
||||
"until its definition changes or `llmgw ctl retry` is run",
|
||||
"model", rep.Model, "failures", n, "last_error", rep.Err)
|
||||
r.notify.Load().Notify(notify.ModelParked, rep.Model,
|
||||
fmt.Sprintf("model %s is parked: it failed %d times in a row (%s); `llmgw ctl retry %s` tries again",
|
||||
rep.Model, n, rep.Err, rep.Model),
|
||||
map[string]any{"model": rep.Model, "failures": n, "last_error": rep.Err})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1216,3 +1228,9 @@ func (r *Reconciler) engineKey(replicaID string) string {
|
||||
}
|
||||
return r.cfg.EngineKey(replicaID)
|
||||
}
|
||||
|
||||
// SetNotifier gives the reconciler somewhere to say a model was parked or a
|
||||
// node lost.
|
||||
func (r *Reconciler) SetNotifier(n *notify.Notifier) {
|
||||
r.notify.Store(n)
|
||||
}
|
||||
Reference in new issue
Block a user