Notifications: webhooks for a parked model, a lost node, a spent budget and new weights; Alertmanager receiver templates
notify.webhooks sends signed JSON or Slack-compatible messages, queued, retried, de-duplicated hourly; ctl notify test checks each webhook. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
646ff996c7
commit
1b71c38f89
24 files changed
+907
-4
No files matched your search
@@ -31,6 +31,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
)
|
||||
|
||||
// Registry reads a repository.
|
||||
@@ -85,6 +86,9 @@ type Deps struct {
|
||||
// UnplacedGrace is how long a model may go unplaced before the job
|
||||
// stops waiting for it: a minute unless set.
|
||||
UnplacedGrace time.Duration
|
||||
// Notify is told when a model's repository publishes new weights; nil
|
||||
// tells no one.
|
||||
Notify *notify.Notifier
|
||||
}
|
||||
|
||||
// errBusy is a model the cluster has the hardware for, all of it in use now.
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
@@ -343,8 +344,16 @@ func (s *Server) checkOne(ctx context.Context, m catalog.Model, o *catalog.Origi
|
||||
}
|
||||
}
|
||||
s.mu.Lock()
|
||||
prev, had := s.updates[m.Name]
|
||||
s.updates[m.Name] = u
|
||||
s.mu.Unlock()
|
||||
// Told once per new revision, not at every check that finds it still there.
|
||||
if u.Available && (!had || !prev.Available || prev.Latest != u.Latest) {
|
||||
s.deps.Notify.Notify(notify.UpdateAvailable, m.Name+"@"+u.Latest,
|
||||
fmt.Sprintf("%s has new weights at %s (%s); `llmgw ctl model update %s` moves to them",
|
||||
m.Name, o.Repo, catalog.ShortRevision(u.Latest), m.Name),
|
||||
map[string]any{"model": m.Name, "repo": o.Repo, "current": o.Revision, "latest": u.Latest})
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,8 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify/notifytest"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -338,3 +340,15 @@ func TestAnOriginGivenByHandIsChecked(t *testing.T) {
|
||||
t.Fatalf("%+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewWeightsAreToldOncePerRevision(t *testing.T) {
|
||||
s, _, _, _, _ := updSetup(t)
|
||||
n, sent := notifytest.Catch(t)
|
||||
s.deps.Notify = n
|
||||
s.CheckUpdates(context.Background())
|
||||
s.CheckUpdates(context.Background()) // still there: not told again
|
||||
got := sent()
|
||||
if len(got) != 1 || got[0].Kind != notify.UpdateAvailable || got[0].Detail["model"] != "qwen" {
|
||||
t.Fatalf("sent %+v, want one update_available for qwen", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user