Setup asks where alerts go, and sends each a test

notify.webhooks existed and setup never asked for one, so a new site
learnt of a parked model or a lost node from its users. `llmgw server
init --webhook URL` (repeatable) and a Notification webhooks field on
both setup screens take them now. Slack's address, Mattermost's and
Rocket.Chat's /hooks/ and Discord's …/slack are sent Slack's text, any
other the event as JSON; slack: or json: in front says which.

Once the control plane runs, they are added to the cluster's settings
beside the webhooks already there, none twice, and setup waits for the
control planes to start again on the change before sending each a test,
saying which answered. A webhook's path is its secret, so it is shown by
host only. A failure is said, not fatal: the cluster serves without its
alerts, and ctl notify test tries again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -28,6 +28,13 @@ APIs change between minors without a deprecation period.
 
 ### Added
 
+- **Setup asks where alerts go**: `llmgw server init --webhook URL`
+  (repeatable) and a **Notification webhooks** field on both setup screens.
+  Slack's, Mattermost's and Rocket.Chat's addresses are sent Slack's text,
+  any other the event as JSON (`slack:` or `json:` in front says which).
+  They are added to the cluster's `notify.webhooks` beside those already
+  there, and each is sent a test once the control plane runs on them.
+
 - **Extended thinking on `/v1/messages`**: `thinking: {type: "enabled"}`
   tells a model that thinks by its chat template to (`enable_thinking`),
   and its reasoning comes back as `thinking` blocks before the text,

docs/install.md

@@ -1827,6 +1827,23 @@ same event about the same thing is sent at most once an hour, so a budget that
 refuses a thousand requests is one message. `llmgw ctl notify test` sends a
 test event to each webhook and says how each answered.
 
+Setup asks for them: **Notification webhooks** on its first page, or
+`--webhook` to `llmgw server init`, as often as there are webhooks:
+
+```
+sudo llmgw server init --admin ops@example.com \
+  --webhook https://hooks.slack.com/services/… \
+  --webhook json:https://ops.example.com/llmgw
+```
+
+An address of Slack's, or one with `/hooks/` in it (Mattermost, Rocket.Chat)
+or ending in `/slack`, is sent Slack's `{"text": …}`; any other the event as
+JSON; `slack:` or `json:` in front says which. They are added to the
+cluster's settings beside any already there — run again, nothing is added
+twice — and once the control plane runs on them, each is sent a test, whose
+answer setup prints. Events to choose and a signing secret are set in the
+settings afterwards (`llmgw ctl config edit`).
+
 ### Tracing
 
 Metrics say that a model took four minutes to start; a trace says where they

internal/cli/serverinit.go

@@ -73,6 +73,9 @@ type serverInit struct {
 	// patroni are user@host of the machines to keep the store on under
 	// Patroni, replicated; this machine is the third etcd vote for two.
 	patroni []string
+	// webhooks are where the cluster's alerts go: put into its settings
+	// once it runs, and each sent a test.
+	webhooks []string
 	// patroniNodes are the same machines already connected to, which the
 	// setup screen does with the password it was given; when set, patroni
 	// only names them.
@@ -182,6 +185,9 @@ func newServerInitCmd() *cobra.Command {
 		"leave the firewall alone, instead of keeping llmgw's internal ports to the cluster's networks")
 	f.BoolVar(&in.postgres, "postgres", false,
 		"keep the store in PostgreSQL on this machine, installing it and moving an existing SQLite store into it")
+	f.StringArrayVar(&in.webhooks, "webhook", nil,
+		"where alerts go — a model parked, a node lost, a budget spent: an http(s) address, Slack's, Mattermost's "+
+			"and Rocket.Chat's sent Slack's text, any other the event as JSON (slack: or json: in front says which); repeatable")
 	f.StringArrayVar(&in.patroni, "patroni", nil,
 		"user@host of a machine to keep the store on, replicated under Patroni; give two or more. "+
 			"Installs PostgreSQL, Patroni and etcd there over ssh, and moves the existing store in")
@@ -249,6 +255,9 @@ func (in *serverInit) run(ctx context.Context) error {
 			fmt.Fprintf(in.out, "  !!    firewall: %v; llmgw's ports answer every network this machine is on\n", err)
 		}
 	}
+	if err := in.ensureWebhooks(ctx); err != nil {
+		fmt.Fprintf(in.out, "  !!    webhooks: %v\n", err)
+	}
 	if in.judgeWanted() && in.configJudgesWith(guardModelName) {
 		// Not fatal: without its judge the baseline policy still checks for
 		// secrets and injection, and a judged request is served flagged.

internal/cli/serverinit_notify.go

@@ -0,0 +1,185 @@
+package cli
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"slices"
+	"strings"
+	"time"
+
+	"go.yaml.in/yaml/v3"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
+)
+
+// Where the cluster says what went wrong: a model parked, a node lost, a
+// budget spent. notify.webhooks existed, and setup never asked for one, so a
+// site learnt of a lost node from its users.
+
+// parseWebhook reads a webhook as setup takes it: an address, with "slack:"
+// or "json:" in front to say what it is sent, else what its address says.
+func parseWebhook(s string) (notify.Webhook, error) {
+	s = strings.TrimSpace(s)
+	format := ""
+	for _, f := range []string{"slack", "json"} {
+		if rest, ok := strings.CutPrefix(s, f+":"); ok && !strings.HasPrefix(rest, "//") {
+			format, s = f, rest
+		}
+	}
+	u, err := url.Parse(s)
+	if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" {
+		return notify.Webhook{}, fmt.Errorf("webhook %q: an http(s) address, with slack: or json: in front to say its format", s)
+	}
+	if format == "" {
+		format = webhookFormat(u)
+	}
+	return notify.Webhook{URL: s, Format: format}, nil
+}
+
+// webhookFormat is what an address takes: Slack's own, and the incoming
+// webhooks that read Slack's {"text": …} — Mattermost's and Rocket.Chat's
+// /hooks/, Discord's …/slack — are sent that; anything else the event as
+// JSON.
+func webhookFormat(u *url.URL) string {
+	switch {
+	case u.Host == "hooks.slack.com",
+		strings.Contains(u.Path, "/hooks/"),
+		strings.HasSuffix(u.Path, "/slack"):
+		return "slack"
+	}
+	return "json"
+}
+
+// parseWebhooks reads a list of them, by commas or as given.
+func parseWebhooks(list []string) ([]notify.Webhook, error) {
+	var out []notify.Webhook
+	for _, item := range list {
+		for _, s := range strings.Split(item, ",") {
+			if strings.TrimSpace(s) == "" {
+				continue
+			}
+			w, err := parseWebhook(s)
+			if err != nil {
+				return nil, err
+			}
+			out = append(out, w)
+		}
+	}
+	return out, nil
+}
+
+// webhookEntry is one of notify.webhooks as the settings document has it.
+type webhookEntry struct {
+	URL        string   `yaml:"url"`
+	Format     string   `yaml:"format,omitempty"`
+	Events     []string `yaml:"events,omitempty,flow"`
+	SecretFile string   `yaml:"secret_file,omitempty"`
+}
+
+// withWebhooks adds these webhooks to a settings document's notify.webhooks,
+// and says whether that changed it: one already there, by address, is left
+// as it is, with whatever events and secret it was given.
+func withWebhooks(doc *settings.Doc, add []notify.Webhook) (bool, error) {
+	path := []string{"notify", "webhooks"}
+	var list []webhookEntry
+	if n := doc.Node(path); n != nil {
+		if err := n.Decode(&list); err != nil {
+			return false, fmt.Errorf("notify.webhooks in the cluster's settings: %w", err)
+		}
+	}
+	changed := false
+	for _, w := range add {
+		if !slices.ContainsFunc(list, func(e webhookEntry) bool { return e.URL == w.URL }) {
+			list = append(list, webhookEntry{URL: w.URL, Format: w.Format})
+			changed = true
+		}
+	}
+	if !changed {
+		return false, nil
+	}
+	b, err := yaml.Marshal(list)
+	if err != nil {
+		return false, err
+	}
+	return true, doc.SetYAML(path, string(b))
+}
+
+// ensureWebhooks puts the webhooks given into the cluster's settings and
+// sends each a test, once the control plane runs on them. Said, not fatal: a
+// cluster without its alerts still serves.
+func (in *serverInit) ensureWebhooks(ctx context.Context) error {
+	if len(in.webhooks) == 0 {
+		return nil
+	}
+	hooks, err := parseWebhooks(in.webhooks)
+	if err != nil {
+		return err
+	}
+	c := in.adminAPI()
+	var cur adminapi.SettingsView
+	if err := c.get(ctx, "/api/v1/settings", &cur); err != nil {
+		return err
+	}
+	doc, err := settings.FromBytes("settings", []byte(cur.Body))
+	if err != nil {
+		return err
+	}
+	changed, err := withWebhooks(doc, hooks)
+	if err != nil {
+		return err
+	}
+	if changed {
+		b, err := doc.Bytes()
+		if err != nil {
+			return err
+		}
+		req, _ := json.Marshal(adminapi.SettingsRequest{Base: cur.Version, Body: string(b), Note: "notification webhooks, from setup"})
+		if err := c.put(ctx, "/api/v1/settings", req, &cur); err != nil {
+			return err
+		}
+		in.done("notification webhooks in the cluster's settings (version %d)", cur.Version)
+	} else {
+		in.kept("notification webhooks (already in the cluster's settings)")
+	}
+
+	// The control planes start again on the change; the test is sent by
+	// one that knows every webhook asked for.
+	var res []notify.Result
+	err = waitFor(ctx, 2*time.Minute, "the control plane to run on its webhooks", func() (bool, error) {
+		res = nil
+		if c.post(ctx, "/api/v1/notify/test", nil, &res) != nil {
+			return false, nil
+		}
+		for _, w := range hooks {
+			// Answered by host: the path of a webhook is its secret.
+			if !slices.ContainsFunc(res, func(r notify.Result) bool { return r.URL == notify.Redact(w.URL) }) {
+				return false, nil
+			}
+		}
+		return true, nil
+	})
+	if err != nil {
+		return err
+	}
+	var failed []string
+	for _, r := range res {
+		if !slices.ContainsFunc(hooks, func(w notify.Webhook) bool { return notify.Redact(w.URL) == r.URL }) {
+			continue
+		}
+		if r.Err != "" {
+			failed = append(failed, r.URL+": "+r.Err)
+			continue
+		}
+		in.done("a test notification reached %s", r.URL)
+	}
+	if len(failed) > 0 {
+		return errors.New("a test notification did not arrive: " + strings.Join(failed, "; ") +
+			" (kept in the settings: 'llmgw ctl notify test' tries again)")
+	}
+	return nil
+}

internal/cli/serverinit_notify_test.go

@@ -0,0 +1,122 @@
+package cli
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
+)
+
+func TestAWebhooksFormatIsWhatItsAddressTakes(t *testing.T) {
+	for in, want := range map[string]string{
+		"https://hooks.slack.com/services/T1/B2/abc":            "slack",
+		"https://chat.example.com/hooks/xyz":                    "slack", // Mattermost, Rocket.Chat
+		"https://discord.com/api/webhooks/1/abc/slack":          "slack",
+		"https://alerts.example.com/llmgw":                      "json",
+		"json:https://hooks.slack.com/services/T1/B2/abc":       "json",
+		"slack:https://alerts.example.com/llmgw":                "slack",
+		"  https://events.pagerduty.com/integration/x/enqueue ": "json",
+	} {
+		w, err := parseWebhook(in)
+		if err != nil || w.Format != want || strings.HasPrefix(w.URL, "slack:") || strings.HasPrefix(w.URL, "json:") {
+			t.Errorf("%q: %+v %v, want %s", in, w, err, want)
+		}
+	}
+	for _, bad := range []string{"hooks.slack.com/x", "ftp://example.com/x", "slack:", "https://"} {
+		if _, err := parseWebhook(bad); err == nil {
+			t.Errorf("%q was taken", bad)
+		}
+	}
+}
+
+func TestWebhooksAreAddedToTheSettingsBesideTheOnesThere(t *testing.T) {
+	doc, err := settings.FromBytes("settings", []byte("router:\n  retries: 2\nnotify:\n  webhooks:\n"+
+		"    - url: https://alerts.example.com/a\n      events: [node_lost]\n      secret_file: /etc/llmgw/secrets/hook\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	hooks, _ := parseWebhooks([]string{"https://alerts.example.com/a, https://hooks.slack.com/services/T/B/c"})
+	changed, err := withWebhooks(doc, hooks)
+	if err != nil || !changed {
+		t.Fatalf("changed %v: %v", changed, err)
+	}
+	b, _ := doc.Bytes()
+	got := string(b)
+	for _, want := range []string{"retries: 2", "url: https://alerts.example.com/a", "events: [node_lost]",
+		"secret_file: /etc/llmgw/secrets/hook", "url: https://hooks.slack.com/services/T/B/c", "format: slack"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("%q not in:\n%s", want, got)
+		}
+	}
+	if strings.Count(got, "alerts.example.com/a") != 1 {
+		t.Errorf("a webhook already there was added again:\n%s", got)
+	}
+	if changed, _ := withWebhooks(doc, hooks); changed {
+		t.Error("adding the same webhooks again changed the settings")
+	}
+}
+
+func TestSetupPutsItsWebhooksInTheSettingsAndTestsThemOnceInForce(t *testing.T) {
+	var mu sync.Mutex
+	body, tests := "router:\n  retries: 2\n", 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mu.Lock()
+		defer mu.Unlock()
+		w.Header().Set("Content-Type", "application/json")
+		switch {
+		case r.Method == http.MethodGet && r.URL.Path == "/api/v1/settings":
+			_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 1, Body: body})
+		case r.Method == http.MethodPut && r.URL.Path == "/api/v1/settings":
+			var req adminapi.SettingsRequest
+			_ = json.NewDecoder(r.Body).Decode(&req)
+			body = req.Body
+			_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 2, Body: body})
+		case r.Method == http.MethodPost && r.URL.Path == "/api/v1/notify/test":
+			tests++
+			// The first answer comes from the control plane before it
+			// started again on the change: it knows no webhooks.
+			res := []notify.Result{}
+			if tests > 1 {
+				res = []notify.Result{{URL: notify.Redact("https://hooks.slack.com/services/T/B/c")},
+					{URL: notify.Redact("https://ops.example.com/in"), Err: "502 Bad Gateway"}}
+			}
+			_ = json.NewEncoder(w).Encode(res)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "server.yaml"),
+		[]byte("admin_listen: "+strings.TrimPrefix(srv.URL, "http://")+"\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	var out bytes.Buffer
+	in := &serverInit{etcDir: dir, out: &out,
+		webhooks: []string{"https://hooks.slack.com/services/T/B/c", "https://ops.example.com/in"}}
+	err := in.ensureWebhooks(t.Context())
+	if err == nil || !strings.Contains(err.Error(), "502") || !strings.Contains(err.Error(), "ctl notify test") {
+		t.Errorf("a webhook that did not answer: %v", err)
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	if !strings.Contains(body, "format: slack") || !strings.Contains(body, "url: https://ops.example.com/in") ||
+		!strings.Contains(body, "retries: 2") || tests < 2 {
+		t.Errorf("settings %q after %d tests", body, tests)
+	}
+	if strings.Contains(out.String(), "/services/T/B/c") {
+		t.Errorf("a webhook's secret path was printed:\n%s", out.String())
+	}
+	if !strings.Contains(out.String(), "reached https://hooks.slack.com/…") {
+		t.Errorf("the one that answered is not said:\n%s", out.String())
+	}
+}

internal/cli/tui_init.go

@@ -50,6 +50,7 @@ func (w *lineWriter) Write(p []byte) (int, error) {
 const (
 	fAdmin = iota
 	fHosts
+	fWebhooks
 	fPostgres
 	fMirror
 	fNode
@@ -70,6 +71,7 @@ type initModel struct {
 	focus     int
 	admin     textField
 	hosts     textField
+	webhooks  textField
 	bundle    textField
 	bundleSHA textField
 	toggles   [fCount]bool
@@ -103,6 +105,11 @@ func newInitModel(in *serverInit, found []string, askForm bool) initModel {
 	hosts.SetValue(strings.Join(in.hosts, ", "))
 	hosts.CharLimit = 400
 	hosts.Width = 40
+	webhooks := newTextField()
+	webhooks.Placeholder = "https://hooks.slack.com/services/…"
+	webhooks.SetValue(strings.Join(in.webhooks, ", "))
+	webhooks.CharLimit = 800
+	webhooks.Width = 40
 	bundle := newTextField()
 	bundle.Placeholder = "/media/usb/llmgw-bundle.tar"
 	bundle.SetValue(in.bundle)
@@ -114,7 +121,7 @@ func newInitModel(in *serverInit, found []string, askForm bool) initModel {
 	bundleSHA.CharLimit = 64
 	bundleSHA.Width = 64
 	sp := newSpinner()
-	m := initModel{in: in, found: found, form: askForm, admin: admin, hosts: hosts,
+	m := initModel{in: in, found: found, form: askForm, admin: admin, hosts: hosts, webhooks: webhooks,
 		bundle: bundle, bundleSHA: bundleSHA, spin: sp, width: 100}
 	m.toggles[fPostgres] = in.postgres
 	m.toggles[fMirror] = !in.noMirror
@@ -235,6 +242,17 @@ func (m initModel) formKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
 			m.moveFocus(0)
 			return m, nil
 		}
+		hooks := strings.TrimSpace(m.webhooks.Value())
+		if _, err := parseWebhooks([]string{hooks}); err != nil {
+			m.formError = err.Error()
+			m.focus = fWebhooks
+			m.moveFocus(0)
+			return m, nil
+		}
+		m.in.webhooks = nil
+		if hooks != "" {
+			m.in.webhooks = []string{hooks}
+		}
 		m.in.bundle, m.in.bundleSHA = bundle, sum
 		m.in.admin = admin
 		m.in.hosts = nil
@@ -260,6 +278,10 @@ func (m initModel) formKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
 		var cmd tea.Cmd
 		m.hosts, cmd = m.hosts.Update(k)
 		return m, cmd
+	case fWebhooks:
+		var cmd tea.Cmd
+		m.webhooks, cmd = m.webhooks.Update(k)
+		return m, cmd
 	case fBundle:
 		var cmd tea.Cmd
 		m.bundle, cmd = m.bundle.Update(k)
@@ -284,6 +306,7 @@ func (m *initModel) moveFocus(d int) {
 	}
 	m.admin.Blur()
 	m.hosts.Blur()
+	m.webhooks.Blur()
 	m.bundle.Blur()
 	m.bundleSHA.Blur()
 	switch m.focus {
@@ -291,6 +314,8 @@ func (m *initModel) moveFocus(d int) {
 		m.admin.Focus()
 	case fHosts:
 		m.hosts.Focus()
+	case fWebhooks:
+		m.webhooks.Focus()
 	case fBundle:
 		m.bundle.Focus()
 	case fBundleSHA:
@@ -409,6 +434,8 @@ func (m initModel) formView() string {
 		found = strings.Join(m.found, ", ")
 	}
 	b.WriteString("    " + sDim.Render("Already in the certificate: "+found+". Add a DNS name or NAT address here.") + "\n\n")
+	fmt.Fprintf(&b, "%s\n    %s\n", label(fWebhooks, "Notification webhooks"), m.webhooks.View())
+	b.WriteString("    " + sDim.Render("Where a parked model, a lost node or a spent budget is said: Slack, Mattermost, or any URL taking JSON. Comma-separated; a test is sent.") + "\n\n")
 	b.WriteString(check(fPostgres, "PostgreSQL", "rather than SQLite; installed here, for several control planes or a larger store"))
 	if m.in.mirrorURL != "" {
 		b.WriteString("  Model mirror " + sDim.Render("at "+m.in.mirrorURL+", from --mirror") + "\n")

internal/cli/tui_init_test.go

@@ -20,6 +20,8 @@ func TestTheInitFormBecomesInitsSettings(t *testing.T) {
 	m = typeIn(m, "ops@example.com")
 	m = send(m, tea.KeyMsg{Type: tea.KeyTab})
 	m = typeIn(m, "llm.example.com, 203.0.113.10")
+	m = send(m, tea.KeyMsg{Type: tea.KeyTab})
+	m = typeIn(m, "https://hooks.slack.com/services/T1/B2/x, json:https://alerts.example.com/in")
 	m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // PostgreSQL on
 	m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab},
 		tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(" ")}) // starter model off
@@ -33,11 +35,30 @@ func TestTheInitFormBecomesInitsSettings(t *testing.T) {
 	if !in.postgres || in.noMirror || in.noNode || !in.noModel || !in.noGuardModel {
 		t.Errorf("toggles: postgres %v mirror %v node %v model %v", in.postgres, !in.noMirror, !in.noNode, !in.noModel)
 	}
+	if hooks, err := parseWebhooks(in.webhooks); err != nil || len(hooks) != 2 || hooks[0].Format != "slack" || hooks[1].Format != "json" {
+		t.Errorf("webhooks %q: %+v %v", in.webhooks, hooks, err)
+	}
 	if m.(initModel).form {
 		t.Error("the form stayed up after Set up")
 	}
 }
 
+func TestAWebhookThatIsNoAddressKeepsTheFormUp(t *testing.T) {
+	in := &serverInit{}
+	var m tea.Model = newInitModel(in, nil, true)
+	for range fWebhooks {
+		m = send(m, tea.KeyMsg{Type: tea.KeyTab})
+	}
+	m = typeIn(m, "hooks.slack.com/services/x")
+	for range fStart - fWebhooks {
+		m = send(m, tea.KeyMsg{Type: tea.KeyTab})
+	}
+	m = send(m, tea.KeyMsg{Type: tea.KeyEnter})
+	if !m.(initModel).form || m.(initModel).focus != fWebhooks || !strings.Contains(m.View(), "http(s) address") {
+		t.Errorf("an address without its scheme was taken:\n%s", m.View())
+	}
+}
+
 func TestAnOfflineBundleIsGivenInTheFormWithItsDigest(t *testing.T) {
 	in := &serverInit{}
 	var m tea.Model = newInitModel(in, nil, true)
@@ -117,9 +138,10 @@ func TestWithAMirrorElsewhereTheFormDoesNotOfferOne(t *testing.T) {
 	if v := m.View(); !strings.Contains(v, "at http://10.0.0.150:8082, from --mirror") {
 		t.Errorf("the mirror in use is not shown:\n%s", v)
 	}
-	// Tab from the addresses goes past the mirror line to PostgreSQL, then
-	// the node.
-	m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab})
+	// Tab from the addresses goes to the webhooks, past the mirror line to
+	// PostgreSQL, then the node.
+	m = send(m, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab}, tea.KeyMsg{Type: tea.KeyTab},
+		tea.KeyMsg{Type: tea.KeyTab})
 	if f := m.(initModel).focus; f != fNode {
 		t.Errorf("focus %d, want the node toggle %d", f, fNode)
 	}

internal/cli/tui_setup.go

@@ -15,6 +15,7 @@ import (
 	"github.com/charmbracelet/lipgloss"
 
 	joinpkg "gitea.mynodes.xyz/ilker/llmgw/internal/join"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/preflight"
@@ -150,8 +151,8 @@ type setupModel struct {
 	ips    []string
 	choice int // 0 one machine, 1 high availability
 
-	admin, names textField
-	afocus       int // admin, names, next
+	admin, names, webhooks textField
+	afocus                 int // admin, names, webhooks, next
 
 	db     [2]dbMachine
 	mfocus int // over both machines' fields, then the check button
@@ -184,8 +185,10 @@ func newSetupModel(host string, ips []string) setupModel {
 	admin.Placeholder, admin.CharLimit, admin.Width = "you@example.com", 200, 40
 	names := newTextField()
 	names.Placeholder, names.CharLimit, names.Width = "llm.example.com, 203.0.113.10", 400, 40
+	webhooks := newTextField()
+	webhooks.Placeholder, webhooks.CharLimit, webhooks.Width = "https://hooks.slack.com/services/…", 800, 40
 	return setupModel{
-		ref: &progRef{}, host: host, ips: ips, admin: admin, names: names,
+		ref: &progRef{}, host: host, ips: ips, admin: admin, names: names, webhooks: webhooks,
 		db: [2]dbMachine{newDBMachine(1), newDBMachine(2)}, spin: newSpinner(), width: 100, height: 30,
 		log: &[]string{}, logMu: &sync.Mutex{},
 	}
@@ -308,14 +311,17 @@ func (m setupModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
 
 func (m setupModel) adminKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
 	move := func(d int) {
-		m.afocus = (m.afocus + d + 3) % 3
+		m.afocus = (m.afocus + d + 4) % 4
 		m.admin.Blur()
 		m.names.Blur()
+		m.webhooks.Blur()
 		switch m.afocus {
 		case 0:
 			m.admin.Focus()
 		case 1:
 			m.names.Focus()
+		case 2:
+			m.webhooks.Focus()
 		}
 	}
 	switch k.Type {
@@ -326,7 +332,7 @@ func (m setupModel) adminKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
 		move(-1)
 		return m, nil
 	case tea.KeyEnter:
-		if m.afocus < 2 {
+		if m.afocus < 3 {
 			move(1)
 			return m, nil
 		}
@@ -336,6 +342,12 @@ func (m setupModel) adminKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
 			move(0)
 			return m, nil
 		}
+		if _, err := parseWebhooks(m.webhookList()); err != nil {
+			m.formErr = err.Error()
+			m.afocus = 2
+			move(0)
+			return m, nil
+		}
 		m.formErr = ""
 		m.page, m.mfocus = pMachines, 0
 		m.db[0].addr.Focus()
@@ -347,10 +359,23 @@ func (m setupModel) adminKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
 		m.admin, cmd = m.admin.Update(k)
 	case 1:
 		m.names, cmd = m.names.Update(k)
+	case 2:
+		m.webhooks, cmd = m.webhooks.Update(k)
 	}
 	return m, cmd
 }
 
+// webhookList is the webhooks asked for, one an item.
+func (m setupModel) webhookList() []string {
+	var out []string
+	for _, s := range strings.Split(m.webhooks.Value(), ",") {
+		if s = strings.TrimSpace(s); s != "" {
+			out = append(out, s)
+		}
+	}
+	return out
+}
+
 func (m *setupModel) focusMachine() {
 	for i := range m.db {
 		for f := 0; f < mfCount; f++ {
@@ -589,6 +614,9 @@ func (m setupModel) command() string {
 	for _, t := range m.targets() {
 		parts = append(parts, "--patroni "+t)
 	}
+	for _, w := range m.webhookList() {
+		parts = append(parts, "--webhook "+w)
+	}
 	parts = append(parts, "--no-node", "--no-model")
 	return strings.Join(parts, " \\\n    ")
 }
@@ -600,6 +628,7 @@ func (m *setupModel) startRun() tea.Cmd {
 	in.admin = strings.TrimSpace(m.admin.Value())
 	in.hosts = m.namesList()
 	in.patroni = m.targets()
+	in.webhooks = m.webhookList()
 	// The control plane of a replicated setup is not where models run;
 	// nodes are added once it is up.
 	in.noNode, in.noModel = true, true
@@ -759,7 +788,9 @@ func (m setupModel) adminView() string {
 		sDim.Render("A password is generated and shown once, at the end."))
 	fmt.Fprintf(&b, "%s\n    %s\n    %s\n\n", label(1, "Other names nodes will reach this machine by"), m.names.View(),
 		sDim.Render("Known: "+strings.Join(append([]string{m.host}, m.ips...), ", ")+". Add a DNS name or a floating address."))
-	b.WriteString(button(m.afocus == 2, "Next") + "\n")
+	fmt.Fprintf(&b, "%s\n    %s\n    %s\n\n", label(2, "Notification webhooks"), m.webhooks.View(),
+		sDim.Render("Where a parked model, a lost node or a spent budget is said: Slack, Mattermost, or any URL taking JSON. Optional; a test is sent."))
+	b.WriteString(button(m.afocus == 3, "Next") + "\n")
 	return b.String()
 }
 
@@ -880,6 +911,13 @@ func (m setupModel) planView() string {
 	fmt.Fprintf(&b, "  %s  %s\n      %s\n", lipgloss.NewStyle().Bold(true).Render(m.host), sDim.Render(m.thisAddr()),
 		"control plane and gateway, model mirror, etcd (the third vote)")
 	fmt.Fprintf(&b, "      administrator %s\n", strings.TrimSpace(m.admin.Value()))
+	if hooks, _ := parseWebhooks(m.webhookList()); len(hooks) > 0 {
+		var where []string
+		for _, h := range hooks {
+			where = append(where, notify.Redact(h.URL)+" ("+h.Format+")")
+		}
+		fmt.Fprintf(&b, "      alerts to %s\n", strings.Join(where, ", "))
+	}
 	granted := false
 	for _, d := range m.db {
 		addr := strings.TrimSpace(d.addr.Value())

internal/cli/tui_setup_test.go

@@ -45,7 +45,7 @@ func TestTheSetupAsksForWhatHighAvailabilityNeedsAndRefusesWhatWillNotDo(t *test
 		t.Fatalf("high availability did not lead to the administrator: page %d", s.page)
 	}
 	// No email: refused, with a reason.
-	m = press(m, "tab", "tab", "enter")
+	m = press(m, "tab", "tab", "tab", "enter")
 	if s := m.(setupModel); s.page != pAdmin || !strings.Contains(s.formErr, "email") {
 		t.Fatalf("went on without an administrator: %+v", s.formErr)
 	}
@@ -54,7 +54,7 @@ func TestTheSetupAsksForWhatHighAvailabilityNeedsAndRefusesWhatWillNotDo(t *test
 	if !strings.Contains(m.View(), "Not yet") {
 		t.Error("the refusal was not shown as a dialog")
 	}
-	m = press(m, "x", "a@example.com", "tab", "tab", "enter")
+	m = press(m, "x", "a@example.com", "tab", "tab", "tab", "enter")
 	s := m.(setupModel)
 	if s.page != pMachines {
 		t.Fatalf("page %d: %s", s.page, s.formErr)

internal/notify/notify.go

@@ -258,7 +258,11 @@ type Result struct {
 
 // redact keeps a webhook's host and drops its path, where Slack and others
 // put the secret.
-func redact(u string) string {
+func redact(u string) string { return Redact(u) }
+
+// Redact is a webhook's address as it may be shown: its host, not the path
+// that is its secret.
+func Redact(u string) string {
 	if i := strings.Index(u, "://"); i >= 0 {
 		if j := strings.IndexByte(u[i+3:], '/'); j >= 0 {
 			return u[:i+3+j] + "/…"