Notifications: webhooks for a parked model, a lost node, a spent budget and new weights; Alertmanager receiver templates
notify.webhooks sends signed JSON or Slack-compatible messages, queued, retried, de-duplicated hourly; ctl notify test checks each webhook. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
646ff996c7
commit
1b71c38f89
24 files changed
+907
-4
No files matched your search
@@ -35,11 +35,14 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// API serves the management endpoints.
|
||||
type API struct {
|
||||
// notifier is the webhooks `ctl notify test` reaches; nil without any.
|
||||
notifier *notify.Notifier
|
||||
// releases is where release binaries are fetched from, for upgrades.
|
||||
releases string
|
||||
// storeStatus reads a replicated store's members; nil for one database.
|
||||
@@ -84,6 +87,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
a.registerGuard(mux)
|
||||
mux.HandleFunc("GET /api/v1/audit", a.auditEvents)
|
||||
mux.HandleFunc("GET /api/v1/usage/export", a.usageExport)
|
||||
mux.HandleFunc("POST /api/v1/notify/test", a.notifyTest)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/adapters", a.setAdapters)
|
||||
mux.HandleFunc("GET /api/v1/models/{name}/target", a.target)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
)
|
||||
|
||||
// SetNotifier gives the API the webhooks `ctl notify test` reaches.
|
||||
func (a *API) SetNotifier(n *notify.Notifier) { a.notifier = n }
|
||||
|
||||
// notifyTest sends a test event to every webhook and says how each answered.
|
||||
func (a *API) notifyTest(w http.ResponseWriter, r *http.Request) {
|
||||
if a.notifier == nil {
|
||||
writeError(w, http.StatusNotFound, "no webhooks: server.yaml has no notify.webhooks")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, a.notifier.Test(r.Context()))
|
||||
}
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
)
|
||||
|
||||
// Registry reads a repository.
|
||||
@@ -85,6 +86,9 @@ type Deps struct {
|
||||
// UnplacedGrace is how long a model may go unplaced before the job
|
||||
// stops waiting for it: a minute unless set.
|
||||
UnplacedGrace time.Duration
|
||||
// Notify is told when a model's repository publishes new weights; nil
|
||||
// tells no one.
|
||||
Notify *notify.Notifier
|
||||
}
|
||||
|
||||
// errBusy is a model the cluster has the hardware for, all of it in use now.
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
@@ -343,8 +344,16 @@ func (s *Server) checkOne(ctx context.Context, m catalog.Model, o *catalog.Origi
|
||||
}
|
||||
}
|
||||
s.mu.Lock()
|
||||
prev, had := s.updates[m.Name]
|
||||
s.updates[m.Name] = u
|
||||
s.mu.Unlock()
|
||||
// Told once per new revision, not at every check that finds it still there.
|
||||
if u.Available && (!had || !prev.Available || prev.Latest != u.Latest) {
|
||||
s.deps.Notify.Notify(notify.UpdateAvailable, m.Name+"@"+u.Latest,
|
||||
fmt.Sprintf("%s has new weights at %s (%s); `llmgw ctl model update %s` moves to them",
|
||||
m.Name, o.Repo, catalog.ShortRevision(u.Latest), m.Name),
|
||||
map[string]any{"model": m.Name, "repo": o.Repo, "current": o.Revision, "latest": u.Latest})
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,8 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify/notifytest"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -338,3 +340,15 @@ func TestAnOriginGivenByHandIsChecked(t *testing.T) {
|
||||
t.Fatalf("%+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewWeightsAreToldOncePerRevision(t *testing.T) {
|
||||
s, _, _, _, _ := updSetup(t)
|
||||
n, sent := notifytest.Catch(t)
|
||||
s.deps.Notify = n
|
||||
s.CheckUpdates(context.Background())
|
||||
s.CheckUpdates(context.Background()) // still there: not told again
|
||||
got := sent()
|
||||
if len(got) != 1 || got[0].Kind != notify.UpdateAvailable || got[0].Detail["model"] != "qwen" {
|
||||
t.Fatalf("sent %+v, want one update_available for qwen", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify/notifytest"
|
||||
)
|
||||
|
||||
func TestParkingAModelTellsSomeone(t *testing.T) {
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
n, sent := notifytest.Catch(t)
|
||||
h.r.SetNotifier(n)
|
||||
failRepeatedly(t, h, DefaultConfig().ParkAfter, "unknown model architecture: 'ggmlc'")
|
||||
h.advance(DefaultConfig().MaxBackoff + time.Second)
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
got := sent()
|
||||
if len(got) != 1 || got[0].Kind != notify.ModelParked || got[0].Subject != "llama" {
|
||||
t.Fatalf("sent %+v, want one model_parked for llama", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLosingANodeTellsSomeone(t *testing.T) {
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
|
||||
n, sent := notifytest.Catch(t)
|
||||
h.r.SetNotifier(n)
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
for _, d := range []time.Duration{20 * time.Second, 90 * time.Second} {
|
||||
h.clock.advance(d)
|
||||
touchAt(h.inv, "2", h.clock.now())
|
||||
h.r.Reconcile(t.Context())
|
||||
}
|
||||
if n, _ := h.inv.Get("1"); n.State != inventory.Lost {
|
||||
t.Fatalf("node 1 is %s", n.State)
|
||||
}
|
||||
got := sent()
|
||||
if len(got) != 1 || got[0].Kind != notify.NodeLost || got[0].Subject != "1" {
|
||||
t.Fatalf("sent %+v, want one node_lost for 1", got)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"math/rand/v2"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
@@ -16,6 +17,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
)
|
||||
|
||||
// ShardCommand is everything an agent needs to launch one shard.
|
||||
@@ -231,6 +233,9 @@ type Reconciler struct {
|
||||
disp Dispatcher
|
||||
log *slog.Logger
|
||||
now func() time.Time
|
||||
// notify tells people about parked models and lost nodes; nil tells
|
||||
// no one.
|
||||
notify atomic.Pointer[notify.Notifier]
|
||||
|
||||
mu sync.Mutex
|
||||
replicas map[string]*Replica
|
||||
@@ -559,6 +564,9 @@ func (r *Reconciler) expireNodes() {
|
||||
r.inv.SetState(n.ID, inventory.Lost)
|
||||
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
|
||||
"node", n.ID, "silent_for", silent.Round(time.Second))
|
||||
r.notify.Load().Notify(notify.NodeLost, n.ID,
|
||||
fmt.Sprintf("node %s was lost: silent for %s; its replicas are rebuilt elsewhere", n.ID, silent.Round(time.Second)),
|
||||
map[string]any{"node": n.ID, "silent_seconds": int(silent.Seconds())})
|
||||
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")
|
||||
}
|
||||
}
|
||||
@@ -644,6 +652,10 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
|
||||
r.log.Error("model parked: it failed every time it started, and is not retried "+
|
||||
"until its definition changes or `llmgw ctl retry` is run",
|
||||
"model", rep.Model, "failures", n, "last_error", rep.Err)
|
||||
r.notify.Load().Notify(notify.ModelParked, rep.Model,
|
||||
fmt.Sprintf("model %s is parked: it failed %d times in a row (%s); `llmgw ctl retry %s` tries again",
|
||||
rep.Model, n, rep.Err, rep.Model),
|
||||
map[string]any{"model": rep.Model, "failures": n, "last_error": rep.Err})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1216,3 +1228,9 @@ func (r *Reconciler) engineKey(replicaID string) string {
|
||||
}
|
||||
return r.cfg.EngineKey(replicaID)
|
||||
}
|
||||
|
||||
// SetNotifier gives the reconciler somewhere to say a model was parked or a
|
||||
// node lost.
|
||||
func (r *Reconciler) SetNotifier(n *notify.Notifier) {
|
||||
r.notify.Store(n)
|
||||
}
|
||||
Reference in new issue
Block a user