From 1d1409faadaba1f7622c3469006d2c5bb35c59c4 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Mon, 5 Oct 2026 02:27:11 +0200 Subject: [PATCH] A standby's admin API is the leader's The console and ctl on a standby showed its own empty view: nodes and replicas live in the leader's memory. A standby now passes /api/ to the leader over the agent port, with the cluster's certificates both ways; the leader answers only a certificate naming a control plane the store lists, and only while it leads. Control plane certificates are issued for client use as well, and server add reissues older ones. An agent's reconnect backoff starts again after a session that worked, so each failover it lives through no longer makes the next one slower. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/install.md | 8 ++ internal/agent/agent.go | 23 +++- internal/cli/leaderforward.go | 168 ++++++++++++++++++++++++++ internal/cli/leaderforward_test.go | 188 +++++++++++++++++++++++++++++ internal/cli/server.go | 17 ++- internal/cli/serveradd.go | 45 ++++++- internal/cli/serverinit.go | 7 +- internal/cli/tui_console.go | 20 ++- internal/cli/tui_setup_test.go | 15 +++ internal/control/adminapi/api.go | 14 ++- internal/control/control.go | 4 + internal/control/leadership.go | 14 +++ 12 files changed, 510 insertions(+), 13 deletions(-) create mode 100644 internal/cli/leaderforward.go create mode 100644 internal/cli/leaderforward_test.go diff --git a/docs/install.md b/docs/install.md index 47ab206..2c0c677 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1940,6 +1940,14 @@ Without it, put a load balancer in front that checks `/ready` — not `CONTROL` line: which leads, which stand by, one that does not answer, and whether the floating address is on the leader. A standby with nothing to take over from is the thing to see before it is needed. +- **Either machine's console.** A standby's admin API passes everything under + `/api/` to the leader, over the port control planes already reach each + other on and with the cluster's certificates both ways, so `sudo llmgw` and + `llmgw ctl` show the same cluster on both — the leader's — and a change made + on the standby is made by the leader. A node's certificate is refused + there. Control plane certificates from before v0.11.3 cannot present + themselves as clients: `server add` reissues them, and the running server + picks the new one up when restarted. - **The mirror** answers only the addresses it was given with `--api-allow`. `server add` checks, and says what to add to the mirror's unit if it refuses the new control plane — which would otherwise find out the first diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 905432d..8e3ebd4 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -21,6 +21,7 @@ import ( "slices" "strconv" "sync" + "sync/atomic" "time" "go.opentelemetry.io/otel/attribute" @@ -42,11 +43,14 @@ import ( // Agent runs on a GPU node. type Agent struct { - cfg Config - log *slog.Logger - sup *supervisor.Supervisor - reg *engines.Registry - http connectHTTPClient + cfg Config + // answered says the current session heard from its control plane, which + // starts the reconnect backoff again when it ends. + answered atomic.Bool + log *slog.Logger + sup *supervisor.Supervisor + reg *engines.Registry + http connectHTTPClient // engineOnce probes the configured engines once before registering. // @@ -215,6 +219,14 @@ func (a *Agent) Run(ctx context.Context) error { // never answers — from here they are the same thing, and trying // somewhere else is the answer to all three. next++ + // A session that was answered was a connection that worked, however + // it ended: the backoff starts again rather than carrying on from + // the last outage. Without this every failover the agent lived + // through made the next one slower — the third waited half a minute + // on a control plane that had been ready for twenty seconds. + if a.answered.Swap(false) { + attempt = 0 + } attempt++ delay := a.reconnectDelay(attempt) // No wait when there is somewhere else to try: the delay exists to @@ -568,6 +580,7 @@ func (a *Agent) session(ctx context.Context, server string) error { // why it is not registering is one an operator cannot fix. return err } + a.answered.Store(true) a.handle(ctx, msg, out) } } diff --git a/internal/cli/leaderforward.go b/internal/cli/leaderforward.go new file mode 100644 index 0000000..46065cb --- /dev/null +++ b/internal/cli/leaderforward.go @@ -0,0 +1,168 @@ +package cli + +import ( + "context" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "net/http/httputil" + "net/url" + "slices" + "strings" + "time" + + "golang.org/x/net/http2" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" +) + +// forwardPrefix is where the leader answers a standby's admin API, on the +// agent port: the one port control planes already reach each other on, with +// the cluster's certificates both ways. The admin port itself listens on +// loopback, and should — it serves pprof. +const forwardPrefix = "/_admin" + +// Headers a forwarded request and its answer carry: which standby asked, and +// which control plane answered. +const ( + hdrForwardedBy = "X-Llmgw-Forwarded-By" + hdrAnsweredBy = "X-Llmgw-Answered-By" +) + +// leaderForward sends a standby's admin API to the leader. +// +// Everything the console and ctl show is the leader's: nodes, replicas and +// jobs live in its memory, put there by the agent streams only it holds. A +// standby answering from its own would show an empty cluster, and accept +// changes it cannot carry out. So a standby answers nothing under /api/ +// itself; the leader does, and the console looks the same on either machine. +type leaderForward struct { + st *store.DB + lead interface{ IsLeader() bool } + self string + tls tlsconf.Config + log *slog.Logger +} + +// standby wraps the admin handler: /api/ goes to the leader while this +// process is not it. +func (f *leaderForward) standby(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasPrefix(r.URL.Path, "/api/") || f.lead.IsLeader() { + next.ServeHTTP(w, r) + return + } + target, name, err := f.leader(r.Context()) + if err != nil { + writeForwardError(w, http.StatusServiceUnavailable, + "this control plane is standing by, and cannot reach the leader: "+err.Error()) + return + } + tc, err := f.tls.ClientConfig(target.Hostname()) + if err != nil { + writeForwardError(w, http.StatusInternalServerError, err.Error()) + return + } + p := &httputil.ReverseProxy{ + Rewrite: func(pr *httputil.ProxyRequest) { + pr.Out.URL.Scheme, pr.Out.URL.Host = "https", target.Host + pr.Out.URL.Path = forwardPrefix + pr.In.URL.Path + pr.Out.URL.RawPath = "" + pr.Out.Host = target.Host + // The leader's own token is not this one's, and the + // certificate is what it believes. + pr.Out.Header.Del("Authorization") + pr.Out.Header.Set(hdrForwardedBy, f.self) + }, + Transport: &http2.Transport{TLSClientConfig: tc}, + // Streams, such as a job's progress, pass through as they come. + FlushInterval: -1, + ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) { + writeForwardError(w, http.StatusBadGateway, fmt.Sprintf("the leader, %s, did not answer: %v", name, err)) + }, + } + p.ServeHTTP(w, r) + }) +} + +// leader is where the lease's holder is reached, and its name. +func (f *leaderForward) leader(ctx context.Context) (*url.URL, string, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + lease, err := f.st.Leadership(ctx, control.ControlRole) + if err != nil || lease.Owner == "" || time.Now().After(lease.ExpiresAt) { + return nil, "", errors.New("no control plane holds the lease right now") + } + name := control.HolderName(lease.Owner) + cps, err := f.st.ControlPlanes(ctx) + if err != nil { + return nil, name, err + } + for _, cp := range cps { + if cp.Name == name { + u, err := url.Parse(cp.URL) + return u, name, err + } + } + return nil, name, fmt.Errorf("%s leads, and is not among the control planes the store lists", name) +} + +// leaderSide answers forwarded requests, on the agent port: only from another +// control plane of this cluster, only while this one leads, and only the API. +func (f *leaderForward) leaderSide(admin http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, forwardPrefix) + if !strings.HasPrefix(path, "/api/") { + http.NotFound(w, r) + return + } + if err := f.fromControlPlane(r); err != nil { + writeForwardError(w, http.StatusForbidden, err.Error()) + return + } + if !f.lead.IsLeader() { + // Not passed on again: two standbys must not bounce a request + // between them while the lease moves. + writeForwardError(w, http.StatusServiceUnavailable, "this control plane no longer leads; ask again") + return + } + r2 := r.Clone(r.Context()) + r2.URL.Path, r2.URL.RawPath = path, "" + w.Header().Set(hdrAnsweredBy, f.self) + admin.ServeHTTP(w, r2) + }) +} + +// fromControlPlane checks the client certificate names a control plane the +// store lists. A node's certificate is from the same authority, and a node +// must not reach the admin API by presenting it. +func (f *leaderForward) fromControlPlane(r *http.Request) error { + if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 { + return errors.New("a control plane's certificate is required") + } + cert := r.TLS.VerifiedChains[0][0] + if !slices.Contains(cert.ExtKeyUsage, x509.ExtKeyUsageServerAuth) { + return errors.New("not a control plane's certificate") + } + cps, err := f.st.ControlPlanes(r.Context()) + if err != nil { + return err + } + for _, cp := range cps { + if cp.Name == cert.Subject.CommonName { + return nil + } + } + return fmt.Errorf("%s is not a control plane of this cluster", cert.Subject.CommonName) +} + +func writeForwardError(w http.ResponseWriter, code int, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]string{"error": msg}) +} diff --git a/internal/cli/leaderforward_test.go b/internal/cli/leaderforward_test.go new file mode 100644 index 0000000..1ccef1f --- /dev/null +++ b/internal/cli/leaderforward_test.go @@ -0,0 +1,188 @@ +package cli + +import ( + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" +) + +type leading struct{ v atomic.Bool } + +func (l *leading) IsLeader() bool { return l.v.Load() } + +// forwardCluster is two control planes of one cluster on one store: a +// leader whose agent port answers forwarded admin requests, and a standby. +type forwardCluster struct { + st *store.DB + dir string + leaderLeads *leading + standby, leader *leaderForward + leaderURL string +} + +func newForwardCluster(t *testing.T) *forwardCluster { + t.Helper() + dir := t.TempDir() + ca, err := tlsconf.NewCA("test CA") + if err != nil { + t.Fatal(err) + } + if err := ca.Write(dir); err != nil { + t.Fatal(err) + } + for _, name := range []string{"ctl-a", "ctl-b"} { + leaf, err := ca.IssuePeer(name, []string{"127.0.0.1"}) + if err != nil { + t.Fatal(err) + } + if err := leaf.Write(dir, name); err != nil { + t.Fatal(err) + } + } + node, err := ca.IssueClient("node1") + if err != nil { + t.Fatal(err) + } + if err := node.Write(dir, "node1"); err != nil { + t.Fatal(err) + } + cfgFor := func(name string) tlsconf.Config { + return tlsconf.Config{CA: filepath.Join(dir, "ca.pem"), Cert: filepath.Join(dir, name+".pem"), Key: filepath.Join(dir, name+"-key.pem")} + } + + st, err := store.Open(t.Context(), filepath.Join(dir, "s.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + + c := &forwardCluster{st: st, dir: dir, leaderLeads: &leading{}} + c.leaderLeads.v.Store(true) + log := slog.New(slog.DiscardHandler) + c.leader = &leaderForward{st: st, lead: c.leaderLeads, self: "ctl-a", tls: cfgFor("ctl-a"), log: log} + standbyLeads := &leading{} + c.standby = &leaderForward{st: st, lead: standbyLeads, self: "ctl-b", tls: cfgFor("ctl-b"), log: log} + + // The leader's admin API: says whose it is, and who it was asked by. + admin := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{"path": r.URL.Path, "by": r.Header.Get(hdrForwardedBy), + "auth": r.Header.Get("Authorization"), "actor": r.Header.Get("X-Llmgw-Actor")}) + }) + mux := http.NewServeMux() + mux.Handle(forwardPrefix+"/", c.leader.leaderSide(admin)) + srv := httptest.NewUnstartedServer(mux) + tc, err := cfgFor("ctl-a").ServerConfig() + if err != nil { + t.Fatal(err) + } + srv.TLS = tc + srv.EnableHTTP2 = true + srv.StartTLS() + t.Cleanup(srv.Close) + c.leaderURL = srv.URL + + for _, cp := range []store.ControlPlane{{URL: srv.URL, Name: "ctl-a"}, {URL: "https://127.0.0.1:1", Name: "ctl-b"}} { + if err := st.PutControlPlane(t.Context(), cp); err != nil { + t.Fatal(err) + } + } + if _, err := st.AcquireLeadership(t.Context(), control.ControlRole, "ctl-a-0123456789abcdef", time.Minute); err != nil { + t.Fatal(err) + } + return c +} + +// ask sends one request through the standby's admin API. +func (c *forwardCluster) ask(t *testing.T, path string) (*http.Response, map[string]string) { + t.Helper() + local := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{"path": "local"}) + }) + srv := httptest.NewServer(c.standby.standby(local)) + defer srv.Close() + req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, srv.URL+path, nil) + req.Header.Set("Authorization", "Bearer the-standbys-own-token") + req.Header.Set("X-Llmgw-Actor", "root@ctl-b") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + var got map[string]string + _ = json.Unmarshal(b, &got) + return resp, got +} + +func TestAStandbysAdminAPIIsTheLeaders(t *testing.T) { + c := newForwardCluster(t) + resp, got := c.ask(t, "/api/v1/summary") + if resp.StatusCode != http.StatusOK || got["path"] != "/api/v1/summary" { + t.Fatalf("%d %v", resp.StatusCode, got) + } + if resp.Header.Get(hdrAnsweredBy) != "ctl-a" || got["by"] != "ctl-b" { + t.Errorf("answered by %q, asked by %q", resp.Header.Get(hdrAnsweredBy), got["by"]) + } + if got["auth"] != "" { + t.Error("the standby's token was passed on to the leader") + } + if got["actor"] != "root@ctl-b" { + t.Errorf("who asked was lost: %q", got["actor"]) + } + // What is not the API stays on the machine: the dashboard, pprof. + if _, got := c.ask(t, "/debug/pprof/"); got["path"] != "local" { + t.Errorf("a non-API path was forwarded: %v", got) + } +} + +func TestTheLeaderAnswersOnlyControlPlanesAndOnlyWhileItLeads(t *testing.T) { + c := newForwardCluster(t) + call := func(cert string) int { + cfg := tlsconf.Config{CA: filepath.Join(c.dir, "ca.pem"), Cert: filepath.Join(c.dir, cert+".pem"), Key: filepath.Join(c.dir, cert+"-key.pem")} + tc, err := cfg.ClientConfig("127.0.0.1") + if err != nil { + t.Fatal(err) + } + tc.NextProtos = nil + client := &http.Client{Transport: &http.Transport{TLSClientConfig: tc}} + resp, err := client.Get(c.leaderURL + forwardPrefix + "/api/v1/summary") + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + return resp.StatusCode + } + if code := call("ctl-b"); code != http.StatusOK { + t.Errorf("a control plane was refused: %d", code) + } + // A node's certificate is from the same authority, and is no way in. + if code := call("node1"); code != http.StatusForbidden { + t.Errorf("a node reached the admin API: %d", code) + } + c.leaderLeads.v.Store(false) + if code := call("ctl-b"); code != http.StatusServiceUnavailable { + t.Errorf("a control plane that no longer leads answered: %d", code) + } +} + +func TestAStandbyWithNoLeaderSaysSo(t *testing.T) { + c := newForwardCluster(t) + if err := c.st.ReleaseLeadership(t.Context(), control.ControlRole, "ctl-a-0123456789abcdef"); err != nil { + t.Fatal(err) + } + resp, got := c.ask(t, "/api/v1/summary") + if resp.StatusCode != http.StatusServiceUnavailable || !strings.Contains(got["error"], "standing by") { + t.Errorf("%d %v", resp.StatusCode, got) + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 342bffd..173810c 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -502,8 +502,20 @@ func newServerCmd() *cobra.Command { if err != nil { return err } + adminHandler := auditAPI(rt, adminMux, log) + // With a rival, a standby's admin API is the leader's: what the + // console and ctl show lives in the leader's memory. + var forward *leaderForward + if admin != nil && admin.Leadership() != nil && rt.Store() != nil && cfg.AgentTLS.Enabled() { + self, _ := os.Hostname() + forward = &leaderForward{st: rt.Store(), lead: admin.Leadership(), self: self, tls: cfg.AgentTLS, log: log} + } + served := adminHandler + if forward != nil { + served = forward.standby(adminHandler) + } adminSrv := withLimits(&http.Server{ - Handler: router.AdminGuard(router.AdminAuth(adminToken, cfg.DebugPprof, auditAPI(rt, adminMux, log))), + Handler: router.AdminGuard(router.AdminAuth(adminToken, cfg.DebugPprof, served)), ReadTimeout: time.Minute, }) @@ -569,6 +581,9 @@ func newServerCmd() *cobra.Command { if agentHandler != nil { mux := http.NewServeMux() mux.Handle(agentPath, agentHandler) + if forward != nil { + mux.Handle(forwardPrefix+"/", router.AdminGuard(forward.leaderSide(adminHandler))) + } // No read timeout: an agent's stream is one request that // lasts as long as the agent does. agentSrv = withLimits(&http.Server{Handler: mux}) diff --git a/internal/cli/serveradd.go b/internal/cli/serveradd.go index 5bae60f..d95bbd1 100644 --- a/internal/cli/serveradd.go +++ b/internal/cli/serveradd.go @@ -121,6 +121,7 @@ func (sa *serverAdd) run(ctx context.Context) error { {"connect", sa.connect}, {"binary", sa.binary}, {"directories", sa.dirs}, + {"this certificate", sa.ownCertificate}, {"certificate", sa.certificate}, {"configuration", sa.configuration}, {"store", sa.storeAccess}, @@ -231,6 +232,40 @@ sudo install -d -o llmgw -g llmgw -m 0750 /var/lib/llmgw`, nil) return err } +// ownCertificate makes this control plane's certificate one it can also +// present as a client — a standby asks the leader for the console's data +// with it — reissuing one from before that with the same names. The running +// server reads it as it starts. +func (sa *serverAdd) ownCertificate(context.Context) error { + path := filepath.Join(sa.etcDir, "server.pem") + cert, err := readCert(path) + if err != nil { + return err + } + if clientCapable(cert) { + sa.kept("this control plane's certificate") + return nil + } + ca, err := loadCA(sa.pkiDir) + if err != nil { + return fmt.Errorf("the authority's key is not in %s: run this where it is (%v)", sa.pkiDir, err) + } + hosts := strings.Split(sanNames(cert), ", ") + leaf, err := ca.IssuePeer(cert.Subject.CommonName, hosts) + if err != nil { + return err + } + if err := os.WriteFile(path, leaf.CertPEM, 0o644); err != nil { //nolint:gosec // a certificate + return err + } + if err := os.WriteFile(filepath.Join(sa.etcDir, "server-key.pem"), leaf.KeyPEM, 0o640); err != nil { + return err + } + sa.done("this control plane's certificate reissued for %s, to ask the leader as well as answer;\n"+ + " restart llmgw-server here for it to take effect — %s takes over meanwhile", strings.Join(hosts, ", "), sa.name) + return nil +} + // certificate issues it a server certificate from this cluster's authority, // for its name and addresses, unless it holds one that already names them. func (sa *serverAdd) certificate(ctx context.Context) error { @@ -255,14 +290,14 @@ func (sa *serverAdd) certificate(ctx context.Context) error { for _, w := range want { missing = missing || !slices.Contains(names, w) } - if !missing { + if !missing && clientCapable(cert) { sa.kept("certificate for %s", strings.Join(names, ", ")) return sa.put(ctx, "/etc/llmgw/ca.pem", "0644", "root:root", ca.PEM) } } } } - leaf, err := ca.IssueServer(sa.name, want) + leaf, err := ca.IssuePeer(sa.name, want) if err != nil { return err } @@ -545,3 +580,9 @@ vrrp_instance llmgw { } `, iface, priority, pass, vip, prefix) } + +// clientCapable says a control plane's certificate can also present itself +// as a client, as a standby does to the leader. +func clientCapable(cert *x509.Certificate) bool { + return slices.Contains(cert.ExtKeyUsage, x509.ExtKeyUsageClientAuth) +} diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index de7015a..d797344 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -462,7 +462,10 @@ func (in *serverInit) ensureCerts(context.Context) error { // A certificate from this authority naming every address is left // alone. One naming fewer is reissued with the union, so an address // that worked yesterday is never dropped by adding one today. - if len(missing) == 0 && cert.CheckSignatureFrom(ca.Cert) == nil { + // And one that can also present itself as a client, which a + // standby's admin API needs to reach the leader's; certificates + // from before that are reissued. + if len(missing) == 0 && cert.CheckSignatureFrom(ca.Cert) == nil && clientCapable(cert) { in.kept("server certificate for %s", strings.Join(have, ", ")) return nil } @@ -477,7 +480,7 @@ func (in *serverInit) ensureCerts(context.Context) error { if name == "" { name = "llmgw-control" } - leaf, err := ca.IssueServer(name, want) + leaf, err := ca.IssuePeer(name, want) if err != nil { return err } diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index 0ac4599..3f15914 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -1065,7 +1065,7 @@ func (m consoleModel) panel() (title, body, keys string) { c := &cobra.Command{} c.SetOut(&b) printSummary(c, m.data.sum) - return "Overview", "\n" + indent(b.String(), " "), keys + return "Overview", m.standbyNote() + "\n" + indent(b.String(), " "), keys case cNodes, cLogs: if m.detail && m.row < len(m.data.nodes) { n := m.data.nodes[m.row] @@ -1569,3 +1569,21 @@ func runConsole(ctx context.Context, out io.Writer, file string) error { const floatingQuestion = "A floating address for clients, kept on whichever control plane leads,\n" + "as 10.0.0.161/24 — a free one on this network. Empty for none, when a load\n" + "balancer in front checks /ready instead." + +// standbyNote says, on a control plane standing by, whose cluster the +// console shows: the leader's, which this one's admin API passes it on to. +func (m consoleModel) standbyNote() string { + var here, leader string + for _, cp := range m.data.sum.ControlPlanes.Members { + if cp.This { + here = cp.State + } + if cp.State == "leading" { + leader = cp.Name + } + } + if here != "standing by" || leader == "" { + return "" + } + return "\n " + sDim.Render("Standing by. Shown, and changed, on the leader: "+leader) + "\n" +} diff --git a/internal/cli/tui_setup_test.go b/internal/cli/tui_setup_test.go index a4d8c7f..fc0b4e5 100644 --- a/internal/cli/tui_setup_test.go +++ b/internal/cli/tui_setup_test.go @@ -203,3 +203,18 @@ func TestTheConsoleAddsAControlPlaneWithAFloatingAddress(t *testing.T) { t.Errorf("dialog %q: %s", c.dialog, c.dialogText) } } + +func TestTheConsoleOnAStandbySaysWhoseClusterItShows(t *testing.T) { + m := consoleModel{menu: cOverview, loaded: true, width: 120, height: 40} + m.data.sum.ControlPlanes = adminapi.ControlPlanes{Members: []adminapi.ControlPlane{ + {Name: "llmgw-ctl", State: "leading"}, + {Name: "llmgw-ctl02", State: "standing by", This: true}, + }} + if v := m.View(); !strings.Contains(v, "Standing by. Shown, and changed, on the leader: llmgw-ctl") { + t.Errorf("the overview on a standby:\n%s", v) + } + m.data.sum.ControlPlanes.Members[0].This, m.data.sum.ControlPlanes.Members[1].This = true, false + if v := m.View(); strings.Contains(v, "Standing by.") { + t.Error("the leader's console says it is standing by") + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 0b50dd5..f95235c 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -1162,8 +1162,18 @@ func (a *API) removeModel(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, Removed{Removed: name}) } -func (a *API) summary(w http.ResponseWriter, _ *http.Request) { - writeJSON(w, http.StatusOK, a.Summary()) +func (a *API) summary(w http.ResponseWriter, r *http.Request) { + s := a.Summary() + // Asked through a standby, which passes on the console's questions to + // the leader: "this one" is the standby the console is on. + if by := r.Header.Get("X-Llmgw-Forwarded-By"); by != "" && len(s.ControlPlanes.Members) > 0 { + members := slices.Clone(s.ControlPlanes.Members) + for i := range members { + members[i].This = members[i].Name == by + } + s.ControlPlanes.Members = members + } + writeJSON(w, http.StatusOK, s) } func (a *API) explainPlacement(w http.ResponseWriter, r *http.Request) { diff --git a/internal/control/control.go b/internal/control/control.go index bf13c85..7630919 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -231,6 +231,10 @@ func (c *Controller) SetLeadership(l *Leadership) { c.rec.SetLeadership(l) } +// Leadership is what SetLeadership set; nil when there is nobody to compete +// with. +func (c *Controller) Leadership() *Leadership { return c.lead } + // IsLeader reports whether this process is placing work. True when leadership // was never configured, because then there is nobody to compete with. func (c *Controller) IsLeader() bool { return c.lead == nil || c.lead.IsLeader() } diff --git a/internal/control/leadership.go b/internal/control/leadership.go index 580d3df..493a6a6 100644 --- a/internal/control/leadership.go +++ b/internal/control/leadership.go @@ -7,6 +7,7 @@ import ( "errors" "log/slog" "os" + "strings" "sync/atomic" "time" @@ -183,3 +184,16 @@ func newOwnerID() string { } return host + "-" + hex.EncodeToString(b[:]) } + +// ControlRole is the lease's name in the store, for readers that ask who +// holds it without competing. +const ControlRole = controlRole + +// HolderName is the machine an owner id names: the hostname it was made +// from, without the random part. +func HolderName(owner string) string { + if i := strings.LastIndexByte(owner, '-'); i > 0 { + return owner[:i] + } + return owner +}