Notifications: webhooks for a parked model, a lost node, a spent budget and new weights; Alertmanager receiver templates

notify.webhooks sends signed JSON or Slack-compatible messages, queued,
retried, de-duplicated hourly; ctl notify test checks each webhook.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 16:58:11 +02:00
1 parent 646ff996c7
commit 1b71c38f89
24 files changed
+907 -4

No files matched your search

@@ -31,6 +31,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
)
// Registry reads a repository.
@@ -85,6 +86,9 @@ type Deps struct {
// UnplacedGrace is how long a model may go unplaced before the job
// stops waiting for it: a minute unless set.
UnplacedGrace time.Duration
// Notify is told when a model's repository publishes new weights; nil
// tells no one.
Notify *notify.Notifier
}
// errBusy is a model the cluster has the hardware for, all of it in use now.
+9
View File
@@ -16,6 +16,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"github.com/prometheus/client_golang/prometheus"
)
@@ -343,8 +344,16 @@ func (s *Server) checkOne(ctx context.Context, m catalog.Model, o *catalog.Origi
}
}
s.mu.Lock()
prev, had := s.updates[m.Name]
s.updates[m.Name] = u
s.mu.Unlock()
// Told once per new revision, not at every check that finds it still there.
if u.Available && (!had || !prev.Available || prev.Latest != u.Latest) {
s.deps.Notify.Notify(notify.UpdateAvailable, m.Name+"@"+u.Latest,
fmt.Sprintf("%s has new weights at %s (%s); `llmgw ctl model update %s` moves to them",
m.Name, o.Repo, catalog.ShortRevision(u.Latest), m.Name),
map[string]any{"model": m.Name, "repo": o.Repo, "current": o.Revision, "latest": u.Latest})
}
return u
}
@@ -14,6 +14,8 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify/notifytest"
)
const (
@@ -338,3 +340,15 @@ func TestAnOriginGivenByHandIsChecked(t *testing.T) {
t.Fatalf("%+v", o)
}
}
func TestNewWeightsAreToldOncePerRevision(t *testing.T) {
s, _, _, _, _ := updSetup(t)
n, sent := notifytest.Catch(t)
s.deps.Notify = n
s.CheckUpdates(context.Background())
s.CheckUpdates(context.Background()) // still there: not told again
got := sent()
if len(got) != 1 || got[0].Kind != notify.UpdateAvailable || got[0].Detail["model"] != "qwen" {
t.Fatalf("sent %+v, want one update_available for qwen", got)
}
}