diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 3b255bb..9cf25a0 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -3443,6 +3443,45 @@ } } }, + "ControlPlane": { + "type": "object", + "description": "ControlPlane is one control plane.", + "properties": { + "name": { + "type": "string" + }, + "address": { + "type": "string" + }, + "state": { + "type": "string", + "description": "State is \"leading\", \"standing by\" or \"not answering\"." + }, + "this": { + "type": "boolean", + "description": "This is the control plane that answered the summary." + } + } + }, + "ControlPlanes": { + "type": "object", + "description": "ControlPlanes is every control plane of a cluster that has several, as each answered when asked.", + "properties": { + "members": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ControlPlane" + } + }, + "floating": { + "type": "string", + "description": "Floating is the address keepalived keeps on the leader (10.0.0.161), and FloatingUp whether it answered as a leader when asked; empty when nothing floats." + }, + "floating_up": { + "type": "boolean" + } + } + }, "DomainView": { "type": "object", "description": "DomainView is one domain: its nodes, its power and whether it is held.", @@ -5276,6 +5315,10 @@ }, "store_error": { "type": "string" + }, + "control_planes": { + "$ref": "#/components/schemas/ControlPlanes", + "description": "ControlPlanes is the cluster's control planes when there is more than one: which leads, which stand by, and the floating address in front." } } }, diff --git a/docs/install.md b/docs/install.md index 668ec5b..47ab206 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1915,15 +1915,41 @@ one trusted machine only; see [development.md](development.md). ## Two control planes -llmgw handles the hard half itself: only one places work, and agents find -whichever is leading. What it does not do is move client traffic, which -happens in front of both. +Once the store is replicated (`server init --patroni`, above), one command on +the control plane makes another machine a second one — or **Do → Add a +control plane** in the console, which asks the same two questions: -[deploy/ha/](../deploy/ha/) has the pattern and a keepalived example. The one -thing to get right is the health check: point it at `/ready`, not `/health`. -`/health` is 200 on a standby, which is alive and holds no endpoints at all, -so a load balancer using it would send half the traffic to something that -refuses every request. +``` +sudo --preserve-env=SSH_AUTH_SOCK llmgw server add ubuntu@10.0.0.150 --vip 10.0.0.161/24 +``` + +Over ssh, as `node add` works, it gives the machine this llmgw, a certificate +from the cluster's authority, this control plane's `server.yaml` and secrets, +and lets it into the store; it starts there as a standby. Every node is told +of it within half a minute and fails over to it without being touched. +`--vip` keeps a floating address on whichever control plane leads, with +keepalived on both: that is the address to give clients, or a proxy in front. +Without it, put a load balancer in front that checks `/ready` — not +`/health`, which is 200 on a standby that would refuse every request. + +- **What a failover costs.** Measured on the reference cluster, stopping the + leader: the standby led within five seconds, every node had moved to it + and every replica was adopted — none restarted, the engines never stopped + serving — and the floating address answered again with it. +- **Who leads.** `llmgw ctl status` and the console's Overview have a + `CONTROL` line: which leads, which stand by, one that does not answer, and + whether the floating address is on the leader. A standby with nothing to + take over from is the thing to see before it is needed. +- **The mirror** answers only the addresses it was given with `--api-allow`. + `server add` checks, and says what to add to the mirror's unit if it + refuses the new control plane — which would otherwise find out the first + time it leads and fetches a model. +- **Limits are per process.** Rate limits are counted in memory, and clients + reach the leader through the floating address, so a failover starts each + key's minute afresh. + +[deploy/ha/](../deploy/ha/) describes what each half does, for anyone putting +their own load balancer in front. ## The portal's language diff --git a/internal/agent/failover_test.go b/internal/agent/failover_test.go index 96a524a..24fe9e4 100644 --- a/internal/agent/failover_test.go +++ b/internal/agent/failover_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" "time" @@ -105,3 +106,49 @@ func TestALeaderStillAcceptsWithNoLeadershipConfigured(t *testing.T) { } t.Fatal("a service with no leadership configured refused an agent") } + +// flipping is a leader that can stop leading partway through a test. +type flipping struct{ leading atomic.Bool } + +func (f *flipping) IsLeader() bool { return f.leading.Load() } + +func TestAnAgentLeavesAControlPlaneThatStopsLeading(t *testing.T) { + // Leadership lost while the process lives on: the store was slow and the + // lease lapsed, or another control plane took it. Agents that stayed on + // the old leader would leave the new one with no nodes, so the old one + // has to let them go rather than wait for them to leave. + was := &flipping{} + was.leading.Store(true) + wasInv, wasAddr := startPlain(t, was) + nowInv, nowAddr := startPlain(t, standby(false)) + + ag := runAgent(t, Config{ + NodeID: "n1", MgmtAddr: "127.0.0.1", + Servers: []string{wasAddr, nowAddr}, + }) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + go ag.Run(ctx) + + waitFor(t, 5*time.Second, "the agent never reached the first leader", func() bool { + _, ok := wasInv.Get("n1") + return ok + }) + was.leading.Store(false) + waitFor(t, 8*time.Second, "the agent stayed on a control plane that stopped leading", func() bool { + _, ok := nowInv.Get("n1") + return ok + }) +} + +func waitFor(t *testing.T, d time.Duration, msg string, ok func() bool) { + t.Helper() + deadline := time.Now().Add(d) + for time.Now().Before(deadline) { + if ok() { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatal(msg) +} diff --git a/internal/cli/controlplanes.go b/internal/cli/controlplanes.go new file mode 100644 index 0000000..f532fed --- /dev/null +++ b/internal/cli/controlplanes.go @@ -0,0 +1,133 @@ +package cli + +import ( + "bufio" + "context" + "net" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// keepalivedPath is where `server add --vip` writes the floating address. +const keepalivedPath = "/etc/keepalived/keepalived.conf" + +// controlPlaneProbe answers the summary's question of which control plane +// leads: it asks each one's /leader, on the client port every one of them +// listens on, since they share a server.yaml. +// +// Asked, rather than read from the lease, because the lease names whoever +// held it last and says nothing about a standby that has stopped — and a +// standby that is not there is the thing worth seeing before it is needed. +type controlPlaneProbe struct { + st *store.DB + self string + floating string + client *http.Client + // leaderURL is where a host is asked; tests point it elsewhere. + leaderURL func(host string) string + + mu sync.Mutex + at time.Time + last adminapi.ControlPlanes +} + +func newControlPlaneProbe(st *store.DB, listen string) *controlPlaneProbe { + _, port, _ := net.SplitHostPort(listen) + if port == "" { + port = "8080" + } + self, _ := os.Hostname() + return &controlPlaneProbe{st: st, self: self, + leaderURL: func(host string) string { return "http://" + net.JoinHostPort(host, port) + "/leader" }, + floating: floatingAddress(keepalivedPath), + client: &http.Client{Timeout: time.Second}} +} + +// Status is the answer, kept for a few seconds: the console asks every three, +// and each asking is a request to every control plane. +func (p *controlPlaneProbe) Status() adminapi.ControlPlanes { + p.mu.Lock() + defer p.mu.Unlock() + if time.Since(p.at) < 2*time.Second { + return p.last + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + cps, err := p.st.ControlPlanes(ctx) + if err != nil || len(cps) < 2 { + // One control plane is not a set of them; nothing to say. + p.at, p.last = time.Now(), adminapi.ControlPlanes{} + return p.last + } + out := adminapi.ControlPlanes{Members: make([]adminapi.ControlPlane, len(cps)), Floating: p.floating} + var wg sync.WaitGroup + for i, cp := range cps { + host := cp.URL + if u, err := url.Parse(cp.URL); err == nil && u.Hostname() != "" { + host = u.Hostname() + } + out.Members[i] = adminapi.ControlPlane{Name: cp.Name, Address: host, This: cp.Name == p.self} + wg.Go(func() { out.Members[i].State = p.ask(ctx, host) }) + } + if p.floating != "" { + wg.Go(func() { out.FloatingUp = p.ask(ctx, p.floating) == "leading" }) + } + wg.Wait() + p.at, p.last = time.Now(), out + return out +} + +// ask is what one address's /leader says. +func (p *controlPlaneProbe) ask(ctx context.Context, host string) string { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.leaderURL(host), nil) + if err != nil { + return "not answering" + } + resp, err := p.client.Do(req) + if err != nil { + return "not answering" + } + _ = resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK: + return "leading" + case http.StatusServiceUnavailable: + return "standing by" + } + return "not answering" +} + +// floatingAddress is the address in the keepalived configuration server add +// wrote, without its prefix; "" when there is none, or the file is someone +// else's. +func floatingAddress(path string) string { + f, err := os.Open(path) //nolint:gosec // a fixed system path + if err != nil { + return "" + } + defer f.Close() + sc := bufio.NewScanner(f) + ours, in := false, false + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + switch { + case strings.HasPrefix(line, "# llmgw:"): + ours = true + case strings.HasPrefix(line, "virtual_ipaddress"): + in = true + case in && line == "}": + return "" + case in && ours && line != "": + addr, _, _ := strings.Cut(strings.Fields(line)[0], "/") + return addr + } + } + return "" +} diff --git a/internal/cli/controlplanes_test.go b/internal/cli/controlplanes_test.go new file mode 100644 index 0000000..179f5c6 --- /dev/null +++ b/internal/cli/controlplanes_test.go @@ -0,0 +1,131 @@ +package cli + +import ( + "bytes" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +func TestStatusSaysWhichControlPlaneLeadsAndWhetherTheOthersStandBy(t *testing.T) { + render := func(s adminapi.Summary) string { + cmd := &cobra.Command{} + var b bytes.Buffer + cmd.SetOut(&b) + printSummary(cmd, s) + return b.String() + } + out := render(adminapi.Summary{ControlPlanes: adminapi.ControlPlanes{ + Members: []adminapi.ControlPlane{ + {Name: "llmgw-ctl", State: "leading", This: true}, + {Name: "llmgw-ctl02", State: "standing by"}, + }, + Floating: "10.0.0.161", FloatingUp: true, + }}) + if !strings.Contains(out, "CONTROL llmgw-ctl leads (this one), llmgw-ctl02 standing by · 10.0.0.161 on the leader") { + t.Errorf("status:\n%s", out) + } + out = render(adminapi.Summary{ControlPlanes: adminapi.ControlPlanes{ + Members: []adminapi.ControlPlane{ + {Name: "llmgw-ctl", State: "not answering"}, + {Name: "llmgw-ctl02", State: "standing by", This: true}, + }, + Floating: "10.0.0.161", + }}) + for _, want := range []string{"NO LEADER", "llmgw-ctl not answering", "10.0.0.161 NOT ANSWERING"} { + if !strings.Contains(out, want) { + t.Errorf("%q missing:\n%s", want, out) + } + } + if out := render(adminapi.Summary{}); strings.Contains(out, "CONTROL") { + t.Error("a single control plane got a control line") + } +} + +func TestTheProbeAsksEachControlPlaneWhetherItLeads(t *testing.T) { + leading := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + defer leading.Close() + standby := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer standby.Close() + gone := httptest.NewServer(http.NotFoundHandler()) + gone.Close() + + st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + for _, cp := range []store.ControlPlane{ + {URL: "https://10.0.0.1:7443", Name: "one"}, + {URL: "https://10.0.0.2:7443", Name: "two"}, + {URL: "https://10.0.0.3:7443", Name: "three"}, + } { + if err := st.PutControlPlane(t.Context(), cp); err != nil { + t.Fatal(err) + } + } + at := map[string]string{"10.0.0.1": leading.URL, "10.0.0.2": standby.URL, "10.0.0.3": gone.URL, "10.0.0.161": leading.URL} + p := newControlPlaneProbe(st, ":8080") + p.self, p.floating = "two", "10.0.0.161" + p.leaderURL = func(host string) string { return at[host] + "/leader" } + + s := p.Status() + got := map[string]adminapi.ControlPlane{} + for _, m := range s.Members { + got[m.Name] = m + } + if got["one"].State != "leading" || got["two"].State != "standing by" || got["three"].State != "not answering" { + t.Errorf("states: %+v", got) + } + if !got["two"].This || got["one"].This || got["one"].Address != "10.0.0.1" { + t.Errorf("which and where: %+v", got) + } + if !s.FloatingUp { + t.Error("a floating address on the leader read as down") + } +} + +func TestOneControlPlaneIsNotASetOfThem(t *testing.T) { + st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if err := st.PutControlPlane(t.Context(), store.ControlPlane{URL: "https://10.0.0.1:7443", Name: "one"}); err != nil { + t.Fatal(err) + } + if s := newControlPlaneProbe(st, ":8080").Status(); len(s.Members) != 0 { + t.Errorf("one control plane reported as a set: %+v", s) + } +} + +func TestTheFloatingAddressIsReadFromWhatServerAddWrote(t *testing.T) { + dir := t.TempDir() + ours := filepath.Join(dir, "ours.conf") + if err := os.WriteFile(ours, []byte(keepalivedConf("eth0", "10.0.0.161", 24, 110, "abcd1234")), 0o600); err != nil { + t.Fatal(err) + } + if got := floatingAddress(ours); got != "10.0.0.161" { + t.Errorf("read %q", got) + } + theirs := filepath.Join(dir, "theirs.conf") + if err := os.WriteFile(theirs, []byte("vrrp_instance x {\n virtual_ipaddress {\n 10.9.9.9/24\n }\n}\n"), 0o600); err != nil { + t.Fatal(err) + } + if got := floatingAddress(theirs); got != "" { + t.Errorf("someone else's keepalived read as ours: %q", got) + } + if got := floatingAddress(filepath.Join(dir, "none")); got != "" { + t.Errorf("no file read as %q", got) + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 821e5f8..342bffd 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -330,6 +330,9 @@ func newServerCmd() *cobra.Command { return out, err }) } + if st := rt.Store(); st != nil { + api.SetControlPlanes(newControlPlaneProbe(st, cfg.Listen).Status) + } adminAPI = api api.SetReleases(cfg.Releases) api.SetNotifier(notifier) @@ -618,7 +621,12 @@ func newServerCmd() *cobra.Command { sctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() if agentSrv != nil { - _ = agentSrv.Shutdown(sctx) + // Closed, not shut down gracefully: an agent's stream + // never goes idle, so Shutdown would wait out its whole + // timeout while every node stayed on a control plane + // that is leaving, and a standby could not be found + // until it gave up. Closed, they move at once. + _ = agentSrv.Close() } if metricsSrv != nil { _ = metricsSrv.Shutdown(sctx) diff --git a/internal/cli/serveradd.go b/internal/cli/serveradd.go index 735110e..004a5da 100644 --- a/internal/cli/serveradd.go +++ b/internal/cli/serveradd.go @@ -132,6 +132,10 @@ func (sa *serverAdd) run(ctx context.Context) error { fn func(context.Context) error }{"firewall", sa.firewall}) } + steps = append(steps, struct { + name string + fn func(context.Context) error + }{"mirror", sa.mirror}) if sa.vip != "" { steps = append(steps, struct { name string @@ -419,6 +423,38 @@ func (sa *serverAdd) service(ctx context.Context) error { } } +// mirror checks that the mirror's management API answers the new control +// plane. It answers only the addresses and certificates it was told to, and +// one told only of the first control plane refuses the second the moment it +// leads: models cannot be fetched, updated or served from the registry. +// +// Said, not fixed: the mirror is its own machine with its own unit, and +// llmgw does not log in to it. +func (sa *serverAdd) mirror(ctx context.Context) error { + cfg, err := config.Load(filepath.Join(sa.etcDir, "server.yaml")) + if err != nil || cfg.Mirror.API == "" { + return nil + } + if cfg.Mirror.TLS.Enabled() { + fmt.Fprintf(sa.out, " note the mirror at %s answers only the certificates it names: add\n"+ + " --api-client %s to its llmgw-mirror unit, or it refuses %s when it leads\n", + cfg.Mirror.API, sa.name, sa.name) + return nil + } + code, _ := sa.sh(ctx, "curl -s -o /dev/null -m 3 -w '%{http_code}' "+shellQuote(strings.TrimRight(cfg.Mirror.API, "/")+"/")+" || true", nil) + switch code = strings.TrimSpace(code); code { + case "403": + fmt.Fprintf(sa.out, " !! the mirror at %s refuses %s: add --api-allow %s to its\n"+ + " llmgw-mirror unit and restart it, or models cannot be fetched while %s leads\n", + cfg.Mirror.API, sa.name, sa.ip, sa.name) + case "", "000": + fmt.Fprintf(sa.out, " !! the mirror at %s does not answer %s\n", cfg.Mirror.API, sa.name) + default: + sa.done("the mirror at %s answers it", cfg.Mirror.API) + } + return nil +} + // floating keeps an address on whichever of the two leads, with keepalived // on both, following /leader. func (sa *serverAdd) floating(ctx context.Context) error { diff --git a/internal/cli/status.go b/internal/cli/status.go index 1bf4f1f..93e4883 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -95,6 +95,39 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) { fmt.Fprintf(out, "\nMODELS %d of %d have the replicas they asked for\n", s.Models.Satisfied, s.Models.Total) + // The control planes, when there are several: who leads, whether the + // others would be there to take over, and the address clients use. + if cps := s.ControlPlanes; len(cps.Members) > 0 { + var parts []string + leader := false + for _, cp := range cps.Members { + p := cp.Name + switch cp.State { + case "leading": + leader = true + p += " leads" + default: + p += " " + cp.State + } + if cp.This { + p += " (this one)" + } + parts = append(parts, p) + } + if !leader { + parts = append([]string{"NO LEADER — nothing is placed"}, parts...) + } + line := strings.Join(parts, ", ") + if cps.Floating != "" { + if cps.FloatingUp { + line += " · " + cps.Floating + " on the leader" + } else { + line += " · " + cps.Floating + " NOT ANSWERING" + } + } + fmt.Fprintf(out, "CONTROL %s\n", line) + } + // The store, when it is a replicated cluster: one line, saying who leads // and whether every other member keeps up. switch { diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index 78c5023..0ac4599 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "io" + "net" "net/url" "os" "os/exec" @@ -53,6 +54,7 @@ const ( cManual cSettings cAddNode + cAddControl cReplicate cServe cUpgrade @@ -63,7 +65,7 @@ const ( ) var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual", - "server.yaml", "Add a node", "Replicate the store", "Serve a model", "Upgrade the cluster", + "server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Upgrade the cluster", "Restart all replicas", "Back up now", "Leave"} // consoleAction is what is done once the screen closes: each runs its own @@ -447,6 +449,14 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { switch m.menu { case cAddNode: return m.ask("Add a node", "user@address of the machine", 1) + case cAddControl: + if !m.storeReplicated() { + m.dialog, m.dialogTitle = "error", "Add a control plane" + m.dialogText = "A second control plane needs a store both can reach, and this one's\n" + + "is on one machine. Replicate the store first (Do → Replicate the store)." + return m, nil + } + return m.ask("Add a control plane", "user@address of the machine", 2) case cReplicate: return m.ask("Replicate the store", "user@address of database machine", 2) case cLeave: @@ -659,19 +669,38 @@ func (m consoleModel) dialogKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { v := strings.TrimSpace(m.prompt.Value()) switch m.dialog { case "target": - if v == "" { + // The floating address is the one answer that may be left empty. + floating := m.menu == cAddControl && len(m.asked) == 1 + if v == "" && !floating { return m, nil } + if floating && v != "" { + if _, _, err := net.ParseCIDR(v); err != nil { + m.dialogText = floatingQuestion + "\n\n" + sBad.Render("not an address with its prefix, like 10.0.0.161/24") + return m, nil + } + } m.asked = append(m.asked, v) m.prompt.SetValue("") if len(m.asked) == int(m.editKind) { - kind := "node" - if m.menu == cReplicate { - kind = "patroni" + switch m.menu { + case cAddControl: + args := []string{"server", "add", m.asked[0]} + if m.asked[1] != "" { + args = append(args, "--vip", m.asked[1]) + } + m.action = consoleAction{kind: "run", targets: args} + case cReplicate: + m.action = consoleAction{kind: "patroni", targets: m.asked} + default: + m.action = consoleAction{kind: "node", targets: m.asked} } - m.action = consoleAction{kind: kind, targets: m.asked} return m, tea.Quit } + if m.menu == cAddControl { + m.dialogText = floatingQuestion + m.prompt.Placeholder = "10.0.0.161/24" + } case "edit": if err := m.doc.Set(m.editPath, m.editKind, m.prompt.Value()); err != nil { m.dialogText = m.editQuestion() + "\n\n" + sBad.Render(err.Error()) @@ -1173,6 +1202,10 @@ func (m consoleModel) panel() (title, body, keys string) { case cAddNode: item = "\n Adds a machine as a node: installs llmgw on it over ssh, checks it,\n gives it a certificate and starts its agent.\n\n " + sDim.Render("On the command line:") + "\n sudo llmgw node add @
" + case cAddControl: + item = "\n Makes another machine a control plane of this cluster: a standby\n that takes over within seconds when this one is lost. Every node is\n told of it; none is touched. With a floating address, keepalived on\n both keeps it on whichever leads — the address to give clients.\n\n " + + sDim.Render("Needs the store replicated first, so both reach it.") + "\n\n " + + sDim.Render("On the command line:") + "\n sudo --preserve-env=SSH_AUTH_SOCK llmgw server add @
--vip
/" case cReplicate: item = "\n Moves the store onto two machines under Patroni, replicated,\n so losing one loses nothing.\n\n " + sDim.Render("On the command line:") + @@ -1531,3 +1564,8 @@ func runConsole(ctx context.Context, out io.Writer, file string) error { _, _ = bufio.NewReader(os.Stdin).ReadString('\n') } } + +// floatingQuestion is the second question when adding a control plane. +const floatingQuestion = "A floating address for clients, kept on whichever control plane leads,\n" + + "as 10.0.0.161/24 — a free one on this network. Empty for none, when a load\n" + + "balancer in front checks /ready instead." diff --git a/internal/cli/tui_setup_test.go b/internal/cli/tui_setup_test.go index 3958ea7..a4d8c7f 100644 --- a/internal/cli/tui_setup_test.go +++ b/internal/cli/tui_setup_test.go @@ -3,6 +3,7 @@ package cli import ( "os" "path/filepath" + "slices" "strings" "testing" @@ -171,3 +172,34 @@ func TestTheConsoleWatchesTheClusterAndEditsItsSettings(t *testing.T) { t.Errorf("server.yaml:\n%s", b) } } + +func TestTheConsoleAddsAControlPlaneWithAFloatingAddress(t *testing.T) { + replicated := consoleModel{menu: cAddControl, loaded: true, width: 100, height: 30} + replicated.data.sum.Store = []adminapi.StoreMember{{Name: "a", Role: "leader"}} + + // The address, then the floating address; the console closes to run it. + var mm tea.Model = replicated + mm = press(mm, "enter", "ubuntu@10.0.0.150", "enter", "10.0.0.161/24", "enter") + want := []string{"server", "add", "ubuntu@10.0.0.150", "--vip", "10.0.0.161/24"} + if a := mm.(consoleModel).action; a.kind != "run" || !slices.Equal(a.targets, want) { + t.Errorf("%+v", a) + } + + // No floating address is an answer too. + mm = press(replicated, "enter", "ubuntu@10.0.0.150", "enter", "enter") + if a := mm.(consoleModel).action; !slices.Equal(a.targets, want[:3]) { + t.Errorf("without one: %+v", a) + } + + // A mistyped one is caught here rather than half way through. + mm = press(replicated, "enter", "ubuntu@10.0.0.150", "enter", "10.0.0.161", "enter") + if c := mm.(consoleModel); c.action.kind != "" || !strings.Contains(c.dialogText, "prefix") { + t.Errorf("an address without its prefix went through: %+v", c.action) + } + + // On a store on one machine it says what to do first. + mm = press(consoleModel{menu: cAddControl, loaded: true, width: 100, height: 30}, "enter") + if c := mm.(consoleModel); c.dialog != "error" || !strings.Contains(c.dialogText, "Replicate the store") { + t.Errorf("dialog %q: %s", c.dialog, c.dialogText) + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index a328e8c..0b50dd5 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -54,6 +54,9 @@ type API struct { releases string // storeStatus reads a replicated store's members; nil for one database. storeStatus func() ([]StoreMember, error) + // controlPlanes says what each control plane answers; nil for a cluster + // of one. + controlPlanes func() ControlPlanes // serve runs registry-to-replica jobs; nil without a mirror and a store. serve *modelserve.Server // history is the Prometheus the console's charts read from. @@ -870,6 +873,10 @@ func (a *API) SetUsage(u Usage) { a.usage = u } // SetStoreStatus supplies the state of a replicated store, for the summary. func (a *API) SetStoreStatus(f func() ([]StoreMember, error)) { a.storeStatus = f } +// SetControlPlanes sets where the summary learns which control plane leads +// and which stand by. +func (a *API) SetControlPlanes(f func() ControlPlanes) { a.controlPlanes = f } + // ScopeUsageView is one group's spend, with its quota when it has one. type ScopeUsageView struct { Scope string `json:"scope"` diff --git a/internal/control/adminapi/summary.go b/internal/control/adminapi/summary.go index f5dfa2b..b9c106b 100644 --- a/internal/control/adminapi/summary.go +++ b/internal/control/adminapi/summary.go @@ -35,6 +35,9 @@ type Summary struct { // StoreError why it could not be read; both empty for a single database. Store []StoreMember `json:"store,omitempty"` StoreError string `json:"store_error,omitempty"` + // ControlPlanes is the cluster's control planes when there is more than + // one: which leads, which stand by, and the floating address in front. + ControlPlanes ControlPlanes `json:"control_planes,omitzero"` } // StoreMember is one member of a replicated store, as its Patroni reports it. @@ -47,6 +50,27 @@ type StoreMember struct { Lag int64 `json:"lag"` } +// ControlPlanes is every control plane of a cluster that has several, as +// each answered when asked. +type ControlPlanes struct { + Members []ControlPlane `json:"members,omitempty"` + // Floating is the address keepalived keeps on the leader (10.0.0.161), + // and FloatingUp whether it answered as a leader when asked; empty when + // nothing floats. + Floating string `json:"floating,omitempty"` + FloatingUp bool `json:"floating_up,omitempty"` +} + +// ControlPlane is one control plane. +type ControlPlane struct { + Name string `json:"name"` + Address string `json:"address"` + // State is "leading", "standing by" or "not answering". + State string `json:"state"` + // This is the control plane that answered the summary. + This bool `json:"this,omitempty"` +} + // NodeTotals counts nodes by the states that need different responses. type NodeTotals struct { Total int `json:"total"` @@ -128,6 +152,9 @@ func (a *API) Summary() Summary { s.StoreError = err.Error() } } + if a.controlPlanes != nil { + s.ControlPlanes = a.controlPlanes() + } nodes := a.Nodes() byClass := map[string]*ClassUse{} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index cf09121..533dc39 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -302,16 +302,58 @@ func (s *Service) Connect( }() defer wg.Wait() + // Read in a goroutine of its own, so the stream can also be ended from + // this side: Receive blocks until the agent sends, and an agent that is + // only heartbeating would keep a stream this process should no longer + // hold for as long as it likes. Returning ends it; the reader's Receive + // then fails and it follows. + read := make(chan struct{}) + go func() { + defer close(read) + s.readLoop(ctx, stream, c, nodeID) + }() + tick := time.NewTicker(leaderCheck) + defer tick.Stop() + for { + select { + case <-read: + return nil + case <-c.closed: + // Replaced by a newer stream from the same node, or the writer + // failed. + return nil + case <-tick.C: + // Leadership lost while the process lives on — the store was + // slow, or another took the lease. A standby holding agents is + // a cluster whose leader cannot reach its nodes, so let them go + // and they find the leader. + if s.leader != nil && !s.leader.IsLeader() { + s.log.Info("no longer leading; letting the agent go", "node", nodeID) + c.close() + return connect.NewError(connect.CodeUnavailable, + errors.New("this control plane is a standby; connect to the leader")) + } + } + } +} + +// leaderCheck is how often an open stream checks that this process still +// leads. +var leaderCheck = time.Second + +func (s *Service) readLoop(ctx context.Context, + stream *connect.BidiStream[llmgwv1.AgentMessage, llmgwv1.ControlMessage], c *conn, nodeID string, +) { for { msg, err := stream.Receive() if err != nil { c.close() if ctx.Err() != nil || errors.Is(err, io.EOF) { s.log.Info("agent disconnected", "node", nodeID) - return nil + return } s.log.Warn("agent stream failed", "node", nodeID, "err", err) - return nil + return } if kv := msg.GetKvEvents(); kv != nil { s.kvEvents(c, kv) diff --git a/internal/control/leadership.go b/internal/control/leadership.go index 178b32d..580d3df 100644 --- a/internal/control/leadership.go +++ b/internal/control/leadership.go @@ -158,6 +158,9 @@ func (l *Leadership) step(ctx context.Context) { l.log.Warn("could not release leadership; a standby will wait it out", "err", err) return } + // Not leading from here on, so nothing in this process goes on acting as + // if it were while it shuts down. + l.leader.Store(false) l.log.Info("released leadership", "owner", l.owner) }