From ec1918b23ba51b4392d54d6f848c9c2783b0ab39 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Mon, 5 Oct 2026 02:17:51 +0200 Subject: [PATCH] Two control planes: a failover in seconds, and the console to set one up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A control plane being stopped held every agent's stream for the thirty seconds of a graceful shutdown, so the standby led within four seconds and had no nodes for thirty. The agent server is now closed, not drained. A control plane that loses its lease while it lives on lets its agents go, rather than keeping nodes the new leader cannot reach. ctl status and the console's Overview say which control plane leads, which stand by, and whether the floating address is on the leader. The console adds a control plane (Do → Add a control plane), asking for the machine and the floating address. server add says when the mirror would refuse the new control plane. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/admin-api.openapi.json | 43 +++++++++ docs/install.md | 42 +++++++-- internal/agent/failover_test.go | 47 ++++++++++ internal/cli/controlplanes.go | 133 +++++++++++++++++++++++++++ internal/cli/controlplanes_test.go | 131 ++++++++++++++++++++++++++ internal/cli/server.go | 10 +- internal/cli/serveradd.go | 36 ++++++++ internal/cli/status.go | 33 +++++++ internal/cli/tui_console.go | 50 ++++++++-- internal/cli/tui_setup_test.go | 32 +++++++ internal/control/adminapi/api.go | 7 ++ internal/control/adminapi/summary.go | 27 ++++++ internal/control/agentsvc/service.go | 46 ++++++++- internal/control/leadership.go | 3 + 14 files changed, 623 insertions(+), 17 deletions(-) create mode 100644 internal/cli/controlplanes.go create mode 100644 internal/cli/controlplanes_test.go diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 3b255bb..9cf25a0 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -3443,6 +3443,45 @@ } } }, + "ControlPlane": { + "type": "object", + "description": "ControlPlane is one control plane.", + "properties": { + "name": { + "type": "string" + }, + "address": { + "type": "string" + }, + "state": { + "type": "string", + "description": "State is \"leading\", \"standing by\" or \"not answering\"." + }, + "this": { + "type": "boolean", + "description": "This is the control plane that answered the summary." + } + } + }, + "ControlPlanes": { + "type": "object", + "description": "ControlPlanes is every control plane of a cluster that has several, as each answered when asked.", + "properties": { + "members": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ControlPlane" + } + }, + "floating": { + "type": "string", + "description": "Floating is the address keepalived keeps on the leader (10.0.0.161), and FloatingUp whether it answered as a leader when asked; empty when nothing floats." + }, + "floating_up": { + "type": "boolean" + } + } + }, "DomainView": { "type": "object", "description": "DomainView is one domain: its nodes, its power and whether it is held.", @@ -5276,6 +5315,10 @@ }, "store_error": { "type": "string" + }, + "control_planes": { + "$ref": "#/components/schemas/ControlPlanes", + "description": "ControlPlanes is the cluster's control planes when there is more than one: which leads, which stand by, and the floating address in front." } } }, diff --git a/docs/install.md b/docs/install.md index 668ec5b..47ab206 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1915,15 +1915,41 @@ one trusted machine only; see [development.md](development.md). ## Two control planes -llmgw handles the hard half itself: only one places work, and agents find -whichever is leading. What it does not do is move client traffic, which -happens in front of both. +Once the store is replicated (`server init --patroni`, above), one command on +the control plane makes another machine a second one — or **Do → Add a +control plane** in the console, which asks the same two questions: -[deploy/ha/](../deploy/ha/) has the pattern and a keepalived example. The one -thing to get right is the health check: point it at `/ready`, not `/health`. -`/health` is 200 on a standby, which is alive and holds no endpoints at all, -so a load balancer using it would send half the traffic to something that -refuses every request. +``` +sudo --preserve-env=SSH_AUTH_SOCK llmgw server add ubuntu@10.0.0.150 --vip 10.0.0.161/24 +``` + +Over ssh, as `node add` works, it gives the machine this llmgw, a certificate +from the cluster's authority, this control plane's `server.yaml` and secrets, +and lets it into the store; it starts there as a standby. Every node is told +of it within half a minute and fails over to it without being touched. +`--vip` keeps a floating address on whichever control plane leads, with +keepalived on both: that is the address to give clients, or a proxy in front. +Without it, put a load balancer in front that checks `/ready` — not +`/health`, which is 200 on a standby that would refuse every request. + +- **What a failover costs.** Measured on the reference cluster, stopping the + leader: the standby led within five seconds, every node had moved to it + and every replica was adopted — none restarted, the engines never stopped + serving — and the floating address answered again with it. +- **Who leads.** `llmgw ctl status` and the console's Overview have a + `CONTROL` line: which leads, which stand by, one that does not answer, and + whether the floating address is on the leader. A standby with nothing to + take over from is the thing to see before it is needed. +- **The mirror** answers only the addresses it was given with `--api-allow`. + `server add` checks, and says what to add to the mirror's unit if it + refuses the new control plane — which would otherwise find out the first + time it leads and fetches a model. +- **Limits are per process.** Rate limits are counted in memory, and clients + reach the leader through the floating address, so a failover starts each + key's minute afresh. + +[deploy/ha/](../deploy/ha/) describes what each half does, for anyone putting +their own load balancer in front. ## The portal's language diff --git a/internal/agent/failover_test.go b/internal/agent/failover_test.go index 96a524a..24fe9e4 100644 --- a/internal/agent/failover_test.go +++ b/internal/agent/failover_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" "time" @@ -105,3 +106,49 @@ func TestALeaderStillAcceptsWithNoLeadershipConfigured(t *testing.T) { } t.Fatal("a service with no leadership configured refused an agent") } + +// flipping is a leader that can stop leading partway through a test. +type flipping struct{ leading atomic.Bool } + +func (f *flipping) IsLeader() bool { return f.leading.Load() } + +func TestAnAgentLeavesAControlPlaneThatStopsLeading(t *testing.T) { + // Leadership lost while the process lives on: the store was slow and the + // lease lapsed, or another control plane took it. Agents that stayed on + // the old leader would leave the new one with no nodes, so the old one + // has to let them go rather than wait for them to leave. + was := &flipping{} + was.leading.Store(true) + wasInv, wasAddr := startPlain(t, was) + nowInv, nowAddr := startPlain(t, standby(false)) + + ag := runAgent(t, Config{ + NodeID: "n1", MgmtAddr: "127.0.0.1", + Servers: []string{wasAddr, nowAddr}, + }) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + go ag.Run(ctx) + + waitFor(t, 5*time.Second, "the agent never reached the first leader", func() bool { + _, ok := wasInv.Get("n1") + return ok + }) + was.leading.Store(false) + waitFor(t, 8*time.Second, "the agent stayed on a control plane that stopped leading", func() bool { + _, ok := nowInv.Get("n1") + return ok + }) +} + +func waitFor(t *testing.T, d time.Duration, msg string, ok func() bool) { + t.Helper() + deadline := time.Now().Add(d) + for time.Now().Before(deadline) { + if ok() { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatal(msg) +} diff --git a/internal/cli/controlplanes.go b/internal/cli/controlplanes.go new file mode 100644 index 0000000..f532fed --- /dev/null +++ b/internal/cli/controlplanes.go @@ -0,0 +1,133 @@ +package cli + +import ( + "bufio" + "context" + "net" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// keepalivedPath is where `server add --vip` writes the floating address. +const keepalivedPath = "/etc/keepalived/keepalived.conf" + +// controlPlaneProbe answers the summary's question of which control plane +// leads: it asks each one's /leader, on the client port every one of them +// listens on, since they share a server.yaml. +// +// Asked, rather than read from the lease, because the lease names whoever +// held it last and says nothing about a standby that has stopped — and a +// standby that is not there is the thing worth seeing before it is needed. +type controlPlaneProbe struct { + st *store.DB + self string + floating string + client *http.Client + // leaderURL is where a host is asked; tests point it elsewhere. + leaderURL func(host string) string + + mu sync.Mutex + at time.Time + last adminapi.ControlPlanes +} + +func newControlPlaneProbe(st *store.DB, listen string) *controlPlaneProbe { + _, port, _ := net.SplitHostPort(listen) + if port == "" { + port = "8080" + } + self, _ := os.Hostname() + return &controlPlaneProbe{st: st, self: self, + leaderURL: func(host string) string { return "http://" + net.JoinHostPort(host, port) + "/leader" }, + floating: floatingAddress(keepalivedPath), + client: &http.Client{Timeout: time.Second}} +} + +// Status is the answer, kept for a few seconds: the console asks every three, +// and each asking is a request to every control plane. +func (p *controlPlaneProbe) Status() adminapi.ControlPlanes { + p.mu.Lock() + defer p.mu.Unlock() + if time.Since(p.at) < 2*time.Second { + return p.last + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + cps, err := p.st.ControlPlanes(ctx) + if err != nil || len(cps) < 2 { + // One control plane is not a set of them; nothing to say. + p.at, p.last = time.Now(), adminapi.ControlPlanes{} + return p.last + } + out := adminapi.ControlPlanes{Members: make([]adminapi.ControlPlane, len(cps)), Floating: p.floating} + var wg sync.WaitGroup + for i, cp := range cps { + host := cp.URL + if u, err := url.Parse(cp.URL); err == nil && u.Hostname() != "" { + host = u.Hostname() + } + out.Members[i] = adminapi.ControlPlane{Name: cp.Name, Address: host, This: cp.Name == p.self} + wg.Go(func() { out.Members[i].State = p.ask(ctx, host) }) + } + if p.floating != "" { + wg.Go(func() { out.FloatingUp = p.ask(ctx, p.floating) == "leading" }) + } + wg.Wait() + p.at, p.last = time.Now(), out + return out +} + +// ask is what one address's /leader says. +func (p *controlPlaneProbe) ask(ctx context.Context, host string) string { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.leaderURL(host), nil) + if err != nil { + return "not answering" + } + resp, err := p.client.Do(req) + if err != nil { + return "not answering" + } + _ = resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK: + return "leading" + case http.StatusServiceUnavailable: + return "standing by" + } + return "not answering" +} + +// floatingAddress is the address in the keepalived configuration server add +// wrote, without its prefix; "" when there is none, or the file is someone +// else's. +func floatingAddress(path string) string { + f, err := os.Open(path) //nolint:gosec // a fixed system path + if err != nil { + return "" + } + defer f.Close() + sc := bufio.NewScanner(f) + ours, in := false, false + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + switch { + case strings.HasPrefix(line, "# llmgw:"): + ours = true + case strings.HasPrefix(line, "virtual_ipaddress"): + in = true + case in && line == "}": + return "" + case in && ours && line != "": + addr, _, _ := strings.Cut(strings.Fields(line)[0], "/") + return addr + } + } + return "" +} diff --git a/internal/cli/controlplanes_test.go b/internal/cli/controlplanes_test.go new file mode 100644 index 0000000..179f5c6 --- /dev/null +++ b/internal/cli/controlplanes_test.go @@ -0,0 +1,131 @@ +package cli + +import ( + "bytes" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +func TestStatusSaysWhichControlPlaneLeadsAndWhetherTheOthersStandBy(t *testing.T) { + render := func(s adminapi.Summary) string { + cmd := &cobra.Command{} + var b bytes.Buffer + cmd.SetOut(&b) + printSummary(cmd, s) + return b.String() + } + out := render(adminapi.Summary{ControlPlanes: adminapi.ControlPlanes{ + Members: []adminapi.ControlPlane{ + {Name: "llmgw-ctl", State: "leading", This: true}, + {Name: "llmgw-ctl02", State: "standing by"}, + }, + Floating: "10.0.0.161", FloatingUp: true, + }}) + if !strings.Contains(out, "CONTROL llmgw-ctl leads (this one), llmgw-ctl02 standing by · 10.0.0.161 on the leader") { + t.Errorf("status:\n%s", out) + } + out = render(adminapi.Summary{ControlPlanes: adminapi.ControlPlanes{ + Members: []adminapi.ControlPlane{ + {Name: "llmgw-ctl", State: "not answering"}, + {Name: "llmgw-ctl02", State: "standing by", This: true}, + }, + Floating: "10.0.0.161", + }}) + for _, want := range []string{"NO LEADER", "llmgw-ctl not answering", "10.0.0.161 NOT ANSWERING"} { + if !strings.Contains(out, want) { + t.Errorf("%q missing:\n%s", want, out) + } + } + if out := render(adminapi.Summary{}); strings.Contains(out, "CONTROL") { + t.Error("a single control plane got a control line") + } +} + +func TestTheProbeAsksEachControlPlaneWhetherItLeads(t *testing.T) { + leading := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + defer leading.Close() + standby := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer standby.Close() + gone := httptest.NewServer(http.NotFoundHandler()) + gone.Close() + + st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + for _, cp := range []store.ControlPlane{ + {URL: "https://10.0.0.1:7443", Name: "one"}, + {URL: "https://10.0.0.2:7443", Name: "two"}, + {URL: "https://10.0.0.3:7443", Name: "three"}, + } { + if err := st.PutControlPlane(t.Context(), cp); err != nil { + t.Fatal(err) + } + } + at := map[string]string{"10.0.0.1": leading.URL, "10.0.0.2": standby.URL, "10.0.0.3": gone.URL, "10.0.0.161": leading.URL} + p := newControlPlaneProbe(st, ":8080") + p.self, p.floating = "two", "10.0.0.161" + p.leaderURL = func(host string) string { return at[host] + "/leader" } + + s := p.Status() + got := map[string]adminapi.ControlPlane{} + for _, m := range s.Members { + got[m.Name] = m + } + if got["one"].State != "leading" || got["two"].State != "standing by" || got["three"].State != "not answering" { + t.Errorf("states: %+v", got) + } + if !got["two"].This || got["one"].This || got["one"].Address != "10.0.0.1" { + t.Errorf("which and where: %+v", got) + } + if !s.FloatingUp { + t.Error("a floating address on the leader read as down") + } +} + +func TestOneControlPlaneIsNotASetOfThem(t *testing.T) { + st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "s.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if err := st.PutControlPlane(t.Context(), store.ControlPlane{URL: "https://10.0.0.1:7443", Name: "one"}); err != nil { + t.Fatal(err) + } + if s := newControlPlaneProbe(st, ":8080").Status(); len(s.Members) != 0 { + t.Errorf("one control plane reported as a set: %+v", s) + } +} + +func TestTheFloatingAddressIsReadFromWhatServerAddWrote(t *testing.T) { + dir := t.TempDir() + ours := filepath.Join(dir, "ours.conf") + if err := os.WriteFile(ours, []byte(keepalivedConf("eth0", "10.0.0.161", 24, 110, "abcd1234")), 0o600); err != nil { + t.Fatal(err) + } + if got := floatingAddress(ours); got != "10.0.0.161" { + t.Errorf("read %q", got) + } + theirs := filepath.Join(dir, "theirs.conf") + if err := os.WriteFile(theirs, []byte("vrrp_instance x {\n virtual_ipaddress {\n 10.9.9.9/24\n }\n}\n"), 0o600); err != nil { + t.Fatal(err) + } + if got := floatingAddress(theirs); got != "" { + t.Errorf("someone else's keepalived read as ours: %q", got) + } + if got := floatingAddress(filepath.Join(dir, "none")); got != "" { + t.Errorf("no file read as %q", got) + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 821e5f8..342bffd 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -330,6 +330,9 @@ func newServerCmd() *cobra.Command { return out, err }) } + if st := rt.Store(); st != nil { + api.SetControlPlanes(newControlPlaneProbe(st, cfg.Listen).Status) + } adminAPI = api api.SetReleases(cfg.Releases) api.SetNotifier(notifier) @@ -618,7 +621,12 @@ func newServerCmd() *cobra.Command { sctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() if agentSrv != nil { - _ = agentSrv.Shutdown(sctx) + // Closed, not shut down gracefully: an agent's stream + // never goes idle, so Shutdown would wait out its whole + // timeout while every node stayed on a control plane + // that is leaving, and a standby could not be found + // until it gave up. Closed, they move at once. + _ = agentSrv.Close() } if metricsSrv != nil { _ = metricsSrv.Shutdown(sctx) diff --git a/internal/cli/serveradd.go b/internal/cli/serveradd.go index 735110e..004a5da 100644 --- a/internal/cli/serveradd.go +++ b/internal/cli/serveradd.go @@ -132,6 +132,10 @@ func (sa *serverAdd) run(ctx context.Context) error { fn func(context.Context) error }{"firewall", sa.firewall}) } + steps = append(steps, struct { + name string + fn func(context.Context) error + }{"mirror", sa.mirror}) if sa.vip != "" { steps = append(steps, struct { name string @@ -419,6 +423,38 @@ func (sa *serverAdd) service(ctx context.Context) error { } } +// mirror checks that the mirror's management API answers the new control +// plane. It answers only the addresses and certificates it was told to, and +// one told only of the first control plane refuses the second the moment it +// leads: models cannot be fetched, updated or served from the registry. +// +// Said, not fixed: the mirror is its own machine with its own unit, and +// llmgw does not log in to it. +func (sa *serverAdd) mirror(ctx context.Context) error { + cfg, err := config.Load(filepath.Join(sa.etcDir, "server.yaml")) + if err != nil || cfg.Mirror.API == "" { + return nil + } + if cfg.Mirror.TLS.Enabled() { + fmt.Fprintf(sa.out, " note the mirror at %s answers only the certificates it names: add\n"+ + " --api-client %s to its llmgw-mirror unit, or it refuses %s when it leads\n", + cfg.Mirror.API, sa.name, sa.name) + return nil + } + code, _ := sa.sh(ctx, "curl -s -o /dev/null -m 3 -w '%{http_code}' "+shellQuote(strings.TrimRight(cfg.Mirror.API, "/")+"/")+" || true", nil) + switch code = strings.TrimSpace(code); code { + case "403": + fmt.Fprintf(sa.out, " !! the mirror at %s refuses %s: add --api-allow %s to its\n"+ + " llmgw-mirror unit and restart it, or models cannot be fetched while %s leads\n", + cfg.Mirror.API, sa.name, sa.ip, sa.name) + case "", "000": + fmt.Fprintf(sa.out, " !! the mirror at %s does not answer %s\n", cfg.Mirror.API, sa.name) + default: + sa.done("the mirror at %s answers it", cfg.Mirror.API) + } + return nil +} + // floating keeps an address on whichever of the two leads, with keepalived // on both, following /leader. func (sa *serverAdd) floating(ctx context.Context) error { diff --git a/internal/cli/status.go b/internal/cli/status.go index 1bf4f1f..93e4883 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -95,6 +95,39 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) { fmt.Fprintf(out, "\nMODELS %d of %d have the replicas they asked for\n", s.Models.Satisfied, s.Models.Total) + // The control planes, when there are several: who leads, whether the + // others would be there to take over, and the address clients use. + if cps := s.ControlPlanes; len(cps.Members) > 0 { + var parts []string + leader := false + for _, cp := range cps.Members { + p := cp.Name + switch cp.State { + case "leading": + leader = true + p += " leads" + default: + p += " " + cp.State + } + if cp.This { + p += " (this one)" + } + parts = append(parts, p) + } + if !leader { + parts = append([]string{"NO LEADER — nothing is placed"}, parts...) + } + line := strings.Join(parts, ", ") + if cps.Floating != "" { + if cps.FloatingUp { + line += " · " + cps.Floating + " on the leader" + } else { + line += " · " + cps.Floating + " NOT ANSWERING" + } + } + fmt.Fprintf(out, "CONTROL %s\n", line) + } + // The store, when it is a replicated cluster: one line, saying who leads // and whether every other member keeps up. switch { diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index 78c5023..0ac4599 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "io" + "net" "net/url" "os" "os/exec" @@ -53,6 +54,7 @@ const ( cManual cSettings cAddNode + cAddControl cReplicate cServe cUpgrade @@ -63,7 +65,7 @@ const ( ) var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual", - "server.yaml", "Add a node", "Replicate the store", "Serve a model", "Upgrade the cluster", + "server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Upgrade the cluster", "Restart all replicas", "Back up now", "Leave"} // consoleAction is what is done once the screen closes: each runs its own @@ -447,6 +449,14 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { switch m.menu { case cAddNode: return m.ask("Add a node", "user@address of the machine", 1) + case cAddControl: + if !m.storeReplicated() { + m.dialog, m.dialogTitle = "error", "Add a control plane" + m.dialogText = "A second control plane needs a store both can reach, and this one's\n" + + "is on one machine. Replicate the store first (Do → Replicate the store)." + return m, nil + } + return m.ask("Add a control plane", "user@address of the machine", 2) case cReplicate: return m.ask("Replicate the store", "user@address of database machine", 2) case cLeave: @@ -659,19 +669,38 @@ func (m consoleModel) dialogKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { v := strings.TrimSpace(m.prompt.Value()) switch m.dialog { case "target": - if v == "" { + // The floating address is the one answer that may be left empty. + floating := m.menu == cAddControl && len(m.asked) == 1 + if v == "" && !floating { return m, nil } + if floating && v != "" { + if _, _, err := net.ParseCIDR(v); err != nil { + m.dialogText = floatingQuestion + "\n\n" + sBad.Render("not an address with its prefix, like 10.0.0.161/24") + return m, nil + } + } m.asked = append(m.asked, v) m.prompt.SetValue("") if len(m.asked) == int(m.editKind) { - kind := "node" - if m.menu == cReplicate { - kind = "patroni" + switch m.menu { + case cAddControl: + args := []string{"server", "add", m.asked[0]} + if m.asked[1] != "" { + args = append(args, "--vip", m.asked[1]) + } + m.action = consoleAction{kind: "run", targets: args} + case cReplicate: + m.action = consoleAction{kind: "patroni", targets: m.asked} + default: + m.action = consoleAction{kind: "node", targets: m.asked} } - m.action = consoleAction{kind: kind, targets: m.asked} return m, tea.Quit } + if m.menu == cAddControl { + m.dialogText = floatingQuestion + m.prompt.Placeholder = "10.0.0.161/24" + } case "edit": if err := m.doc.Set(m.editPath, m.editKind, m.prompt.Value()); err != nil { m.dialogText = m.editQuestion() + "\n\n" + sBad.Render(err.Error()) @@ -1173,6 +1202,10 @@ func (m consoleModel) panel() (title, body, keys string) { case cAddNode: item = "\n Adds a machine as a node: installs llmgw on it over ssh, checks it,\n gives it a certificate and starts its agent.\n\n " + sDim.Render("On the command line:") + "\n sudo llmgw node add @
" + case cAddControl: + item = "\n Makes another machine a control plane of this cluster: a standby\n that takes over within seconds when this one is lost. Every node is\n told of it; none is touched. With a floating address, keepalived on\n both keeps it on whichever leads — the address to give clients.\n\n " + + sDim.Render("Needs the store replicated first, so both reach it.") + "\n\n " + + sDim.Render("On the command line:") + "\n sudo --preserve-env=SSH_AUTH_SOCK llmgw server add @
--vip
/" case cReplicate: item = "\n Moves the store onto two machines under Patroni, replicated,\n so losing one loses nothing.\n\n " + sDim.Render("On the command line:") + @@ -1531,3 +1564,8 @@ func runConsole(ctx context.Context, out io.Writer, file string) error { _, _ = bufio.NewReader(os.Stdin).ReadString('\n') } } + +// floatingQuestion is the second question when adding a control plane. +const floatingQuestion = "A floating address for clients, kept on whichever control plane leads,\n" + + "as 10.0.0.161/24 — a free one on this network. Empty for none, when a load\n" + + "balancer in front checks /ready instead." diff --git a/internal/cli/tui_setup_test.go b/internal/cli/tui_setup_test.go index 3958ea7..a4d8c7f 100644 --- a/internal/cli/tui_setup_test.go +++ b/internal/cli/tui_setup_test.go @@ -3,6 +3,7 @@ package cli import ( "os" "path/filepath" + "slices" "strings" "testing" @@ -171,3 +172,34 @@ func TestTheConsoleWatchesTheClusterAndEditsItsSettings(t *testing.T) { t.Errorf("server.yaml:\n%s", b) } } + +func TestTheConsoleAddsAControlPlaneWithAFloatingAddress(t *testing.T) { + replicated := consoleModel{menu: cAddControl, loaded: true, width: 100, height: 30} + replicated.data.sum.Store = []adminapi.StoreMember{{Name: "a", Role: "leader"}} + + // The address, then the floating address; the console closes to run it. + var mm tea.Model = replicated + mm = press(mm, "enter", "ubuntu@10.0.0.150", "enter", "10.0.0.161/24", "enter") + want := []string{"server", "add", "ubuntu@10.0.0.150", "--vip", "10.0.0.161/24"} + if a := mm.(consoleModel).action; a.kind != "run" || !slices.Equal(a.targets, want) { + t.Errorf("%+v", a) + } + + // No floating address is an answer too. + mm = press(replicated, "enter", "ubuntu@10.0.0.150", "enter", "enter") + if a := mm.(consoleModel).action; !slices.Equal(a.targets, want[:3]) { + t.Errorf("without one: %+v", a) + } + + // A mistyped one is caught here rather than half way through. + mm = press(replicated, "enter", "ubuntu@10.0.0.150", "enter", "10.0.0.161", "enter") + if c := mm.(consoleModel); c.action.kind != "" || !strings.Contains(c.dialogText, "prefix") { + t.Errorf("an address without its prefix went through: %+v", c.action) + } + + // On a store on one machine it says what to do first. + mm = press(consoleModel{menu: cAddControl, loaded: true, width: 100, height: 30}, "enter") + if c := mm.(consoleModel); c.dialog != "error" || !strings.Contains(c.dialogText, "Replicate the store") { + t.Errorf("dialog %q: %s", c.dialog, c.dialogText) + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index a328e8c..0b50dd5 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -54,6 +54,9 @@ type API struct { releases string // storeStatus reads a replicated store's members; nil for one database. storeStatus func() ([]StoreMember, error) + // controlPlanes says what each control plane answers; nil for a cluster + // of one. + controlPlanes func() ControlPlanes // serve runs registry-to-replica jobs; nil without a mirror and a store. serve *modelserve.Server // history is the Prometheus the console's charts read from. @@ -870,6 +873,10 @@ func (a *API) SetUsage(u Usage) { a.usage = u } // SetStoreStatus supplies the state of a replicated store, for the summary. func (a *API) SetStoreStatus(f func() ([]StoreMember, error)) { a.storeStatus = f } +// SetControlPlanes sets where the summary learns which control plane leads +// and which stand by. +func (a *API) SetControlPlanes(f func() ControlPlanes) { a.controlPlanes = f } + // ScopeUsageView is one group's spend, with its quota when it has one. type ScopeUsageView struct { Scope string `json:"scope"` diff --git a/internal/control/adminapi/summary.go b/internal/control/adminapi/summary.go index f5dfa2b..b9c106b 100644 --- a/internal/control/adminapi/summary.go +++ b/internal/control/adminapi/summary.go @@ -35,6 +35,9 @@ type Summary struct { // StoreError why it could not be read; both empty for a single database. Store []StoreMember `json:"store,omitempty"` StoreError string `json:"store_error,omitempty"` + // ControlPlanes is the cluster's control planes when there is more than + // one: which leads, which stand by, and the floating address in front. + ControlPlanes ControlPlanes `json:"control_planes,omitzero"` } // StoreMember is one member of a replicated store, as its Patroni reports it. @@ -47,6 +50,27 @@ type StoreMember struct { Lag int64 `json:"lag"` } +// ControlPlanes is every control plane of a cluster that has several, as +// each answered when asked. +type ControlPlanes struct { + Members []ControlPlane `json:"members,omitempty"` + // Floating is the address keepalived keeps on the leader (10.0.0.161), + // and FloatingUp whether it answered as a leader when asked; empty when + // nothing floats. + Floating string `json:"floating,omitempty"` + FloatingUp bool `json:"floating_up,omitempty"` +} + +// ControlPlane is one control plane. +type ControlPlane struct { + Name string `json:"name"` + Address string `json:"address"` + // State is "leading", "standing by" or "not answering". + State string `json:"state"` + // This is the control plane that answered the summary. + This bool `json:"this,omitempty"` +} + // NodeTotals counts nodes by the states that need different responses. type NodeTotals struct { Total int `json:"total"` @@ -128,6 +152,9 @@ func (a *API) Summary() Summary { s.StoreError = err.Error() } } + if a.controlPlanes != nil { + s.ControlPlanes = a.controlPlanes() + } nodes := a.Nodes() byClass := map[string]*ClassUse{} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index cf09121..533dc39 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -302,16 +302,58 @@ func (s *Service) Connect( }() defer wg.Wait() + // Read in a goroutine of its own, so the stream can also be ended from + // this side: Receive blocks until the agent sends, and an agent that is + // only heartbeating would keep a stream this process should no longer + // hold for as long as it likes. Returning ends it; the reader's Receive + // then fails and it follows. + read := make(chan struct{}) + go func() { + defer close(read) + s.readLoop(ctx, stream, c, nodeID) + }() + tick := time.NewTicker(leaderCheck) + defer tick.Stop() + for { + select { + case <-read: + return nil + case <-c.closed: + // Replaced by a newer stream from the same node, or the writer + // failed. + return nil + case <-tick.C: + // Leadership lost while the process lives on — the store was + // slow, or another took the lease. A standby holding agents is + // a cluster whose leader cannot reach its nodes, so let them go + // and they find the leader. + if s.leader != nil && !s.leader.IsLeader() { + s.log.Info("no longer leading; letting the agent go", "node", nodeID) + c.close() + return connect.NewError(connect.CodeUnavailable, + errors.New("this control plane is a standby; connect to the leader")) + } + } + } +} + +// leaderCheck is how often an open stream checks that this process still +// leads. +var leaderCheck = time.Second + +func (s *Service) readLoop(ctx context.Context, + stream *connect.BidiStream[llmgwv1.AgentMessage, llmgwv1.ControlMessage], c *conn, nodeID string, +) { for { msg, err := stream.Receive() if err != nil { c.close() if ctx.Err() != nil || errors.Is(err, io.EOF) { s.log.Info("agent disconnected", "node", nodeID) - return nil + return } s.log.Warn("agent stream failed", "node", nodeID, "err", err) - return nil + return } if kv := msg.GetKvEvents(); kv != nil { s.kvEvents(c, kv) diff --git a/internal/control/leadership.go b/internal/control/leadership.go index 178b32d..580d3df 100644 --- a/internal/control/leadership.go +++ b/internal/control/leadership.go @@ -158,6 +158,9 @@ func (l *Leadership) step(ctx context.Context) { l.log.Warn("could not release leadership; a standby will wait it out", "err", err) return } + // Not leading from here on, so nothing in this process goes on acting as + // if it were while it shuts down. + l.leader.Store(false) l.log.Info("released leadership", "owner", l.owner) }