diff --git a/docs/install.md b/docs/install.md index 47ab206..2c0c677 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1940,6 +1940,14 @@ Without it, put a load balancer in front that checks `/ready` — not `CONTROL` line: which leads, which stand by, one that does not answer, and whether the floating address is on the leader. A standby with nothing to take over from is the thing to see before it is needed. +- **Either machine's console.** A standby's admin API passes everything under + `/api/` to the leader, over the port control planes already reach each + other on and with the cluster's certificates both ways, so `sudo llmgw` and + `llmgw ctl` show the same cluster on both — the leader's — and a change made + on the standby is made by the leader. A node's certificate is refused + there. Control plane certificates from before v0.11.3 cannot present + themselves as clients: `server add` reissues them, and the running server + picks the new one up when restarted. - **The mirror** answers only the addresses it was given with `--api-allow`. `server add` checks, and says what to add to the mirror's unit if it refuses the new control plane — which would otherwise find out the first diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 905432d..8e3ebd4 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -21,6 +21,7 @@ import ( "slices" "strconv" "sync" + "sync/atomic" "time" "go.opentelemetry.io/otel/attribute" @@ -42,11 +43,14 @@ import ( // Agent runs on a GPU node. type Agent struct { - cfg Config - log *slog.Logger - sup *supervisor.Supervisor - reg *engines.Registry - http connectHTTPClient + cfg Config + // answered says the current session heard from its control plane, which + // starts the reconnect backoff again when it ends. + answered atomic.Bool + log *slog.Logger + sup *supervisor.Supervisor + reg *engines.Registry + http connectHTTPClient // engineOnce probes the configured engines once before registering. // @@ -215,6 +219,14 @@ func (a *Agent) Run(ctx context.Context) error { // never answers — from here they are the same thing, and trying // somewhere else is the answer to all three. next++ + // A session that was answered was a connection that worked, however + // it ended: the backoff starts again rather than carrying on from + // the last outage. Without this every failover the agent lived + // through made the next one slower — the third waited half a minute + // on a control plane that had been ready for twenty seconds. + if a.answered.Swap(false) { + attempt = 0 + } attempt++ delay := a.reconnectDelay(attempt) // No wait when there is somewhere else to try: the delay exists to @@ -568,6 +580,7 @@ func (a *Agent) session(ctx context.Context, server string) error { // why it is not registering is one an operator cannot fix. return err } + a.answered.Store(true) a.handle(ctx, msg, out) } } diff --git a/internal/cli/leaderforward.go b/internal/cli/leaderforward.go new file mode 100644 index 0000000..46065cb --- /dev/null +++ b/internal/cli/leaderforward.go @@ -0,0 +1,168 @@ +package cli + +import ( + "context" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "net/http/httputil" + "net/url" + "slices" + "strings" + "time" + + "golang.org/x/net/http2" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" +) + +// forwardPrefix is where the leader answers a standby's admin API, on the +// agent port: the one port control planes already reach each other on, with +// the cluster's certificates both ways. The admin port itself listens on +// loopback, and should — it serves pprof. +const forwardPrefix = "/_admin" + +// Headers a forwarded request and its answer carry: which standby asked, and +// which control plane answered. +const ( + hdrForwardedBy = "X-Llmgw-Forwarded-By" + hdrAnsweredBy = "X-Llmgw-Answered-By" +) + +// leaderForward sends a standby's admin API to the leader. +// +// Everything the console and ctl show is the leader's: nodes, replicas and +// jobs live in its memory, put there by the agent streams only it holds. A +// standby answering from its own would show an empty cluster, and accept +// changes it cannot carry out. So a standby answers nothing under /api/ +// itself; the leader does, and the console looks the same on either machine. +type leaderForward struct { + st *store.DB + lead interface{ IsLeader() bool } + self string + tls tlsconf.Config + log *slog.Logger +} + +// standby wraps the admin handler: /api/ goes to the leader while this +// process is not it. +func (f *leaderForward) standby(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasPrefix(r.URL.Path, "/api/") || f.lead.IsLeader() { + next.ServeHTTP(w, r) + return + } + target, name, err := f.leader(r.Context()) + if err != nil { + writeForwardError(w, http.StatusServiceUnavailable, + "this control plane is standing by, and cannot reach the leader: "+err.Error()) + return + } + tc, err := f.tls.ClientConfig(target.Hostname()) + if err != nil { + writeForwardError(w, http.StatusInternalServerError, err.Error()) + return + } + p := &httputil.ReverseProxy{ + Rewrite: func(pr *httputil.ProxyRequest) { + pr.Out.URL.Scheme, pr.Out.URL.Host = "https", target.Host + pr.Out.URL.Path = forwardPrefix + pr.In.URL.Path + pr.Out.URL.RawPath = "" + pr.Out.Host = target.Host + // The leader's own token is not this one's, and the + // certificate is what it believes. + pr.Out.Header.Del("Authorization") + pr.Out.Header.Set(hdrForwardedBy, f.self) + }, + Transport: &http2.Transport{TLSClientConfig: tc}, + // Streams, such as a job's progress, pass through as they come. + FlushInterval: -1, + ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) { + writeForwardError(w, http.StatusBadGateway, fmt.Sprintf("the leader, %s, did not answer: %v", name, err)) + }, + } + p.ServeHTTP(w, r) + }) +} + +// leader is where the lease's holder is reached, and its name. +func (f *leaderForward) leader(ctx context.Context) (*url.URL, string, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + lease, err := f.st.Leadership(ctx, control.ControlRole) + if err != nil || lease.Owner == "" || time.Now().After(lease.ExpiresAt) { + return nil, "", errors.New("no control plane holds the lease right now") + } + name := control.HolderName(lease.Owner) + cps, err := f.st.ControlPlanes(ctx) + if err != nil { + return nil, name, err + } + for _, cp := range cps { + if cp.Name == name { + u, err := url.Parse(cp.URL) + return u, name, err + } + } + return nil, name, fmt.Errorf("%s leads, and is not among the control planes the store lists", name) +} + +// leaderSide answers forwarded requests, on the agent port: only from another +// control plane of this cluster, only while this one leads, and only the API. +func (f *leaderForward) leaderSide(admin http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, forwardPrefix) + if !strings.HasPrefix(path, "/api/") { + http.NotFound(w, r) + return + } + if err := f.fromControlPlane(r); err != nil { + writeForwardError(w, http.StatusForbidden, err.Error()) + return + } + if !f.lead.IsLeader() { + // Not passed on again: two standbys must not bounce a request + // between them while the lease moves. + writeForwardError(w, http.StatusServiceUnavailable, "this control plane no longer leads; ask again") + return + } + r2 := r.Clone(r.Context()) + r2.URL.Path, r2.URL.RawPath = path, "" + w.Header().Set(hdrAnsweredBy, f.self) + admin.ServeHTTP(w, r2) + }) +} + +// fromControlPlane checks the client certificate names a control plane the +// store lists. A node's certificate is from the same authority, and a node +// must not reach the admin API by presenting it. +func (f *leaderForward) fromControlPlane(r *http.Request) error { + if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 { + return errors.New("a control plane's certificate is required") + } + cert := r.TLS.VerifiedChains[0][0] + if !slices.Contains(cert.ExtKeyUsage, x509.ExtKeyUsageServerAuth) { + return errors.New("not a control plane's certificate") + } + cps, err := f.st.ControlPlanes(r.Context()) + if err != nil { + return err + } + for _, cp := range cps { + if cp.Name == cert.Subject.CommonName { + return nil + } + } + return fmt.Errorf("%s is not a control plane of this cluster", cert.Subject.CommonName) +} + +func writeForwardError(w http.ResponseWriter, code int, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]string{"error": msg}) +} diff --git a/internal/cli/leaderforward_test.go b/internal/cli/leaderforward_test.go new file mode 100644 index 0000000..1ccef1f --- /dev/null +++ b/internal/cli/leaderforward_test.go @@ -0,0 +1,188 @@ +package cli + +import ( + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" +) + +type leading struct{ v atomic.Bool } + +func (l *leading) IsLeader() bool { return l.v.Load() } + +// forwardCluster is two control planes of one cluster on one store: a +// leader whose agent port answers forwarded admin requests, and a standby. +type forwardCluster struct { + st *store.DB + dir string + leaderLeads *leading + standby, leader *leaderForward + leaderURL string +} + +func newForwardCluster(t *testing.T) *forwardCluster { + t.Helper() + dir := t.TempDir() + ca, err := tlsconf.NewCA("test CA") + if err != nil { + t.Fatal(err) + } + if err := ca.Write(dir); err != nil { + t.Fatal(err) + } + for _, name := range []string{"ctl-a", "ctl-b"} { + leaf, err := ca.IssuePeer(name, []string{"127.0.0.1"}) + if err != nil { + t.Fatal(err) + } + if err := leaf.Write(dir, name); err != nil { + t.Fatal(err) + } + } + node, err := ca.IssueClient("node1") + if err != nil { + t.Fatal(err) + } + if err := node.Write(dir, "node1"); err != nil { + t.Fatal(err) + } + cfgFor := func(name string) tlsconf.Config { + return tlsconf.Config{CA: filepath.Join(dir, "ca.pem"), Cert: filepath.Join(dir, name+".pem"), Key: filepath.Join(dir, name+"-key.pem")} + } + + st, err := store.Open(t.Context(), filepath.Join(dir, "s.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + + c := &forwardCluster{st: st, dir: dir, leaderLeads: &leading{}} + c.leaderLeads.v.Store(true) + log := slog.New(slog.DiscardHandler) + c.leader = &leaderForward{st: st, lead: c.leaderLeads, self: "ctl-a", tls: cfgFor("ctl-a"), log: log} + standbyLeads := &leading{} + c.standby = &leaderForward{st: st, lead: standbyLeads, self: "ctl-b", tls: cfgFor("ctl-b"), log: log} + + // The leader's admin API: says whose it is, and who it was asked by. + admin := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{"path": r.URL.Path, "by": r.Header.Get(hdrForwardedBy), + "auth": r.Header.Get("Authorization"), "actor": r.Header.Get("X-Llmgw-Actor")}) + }) + mux := http.NewServeMux() + mux.Handle(forwardPrefix+"/", c.leader.leaderSide(admin)) + srv := httptest.NewUnstartedServer(mux) + tc, err := cfgFor("ctl-a").ServerConfig() + if err != nil { + t.Fatal(err) + } + srv.TLS = tc + srv.EnableHTTP2 = true + srv.StartTLS() + t.Cleanup(srv.Close) + c.leaderURL = srv.URL + + for _, cp := range []store.ControlPlane{{URL: srv.URL, Name: "ctl-a"}, {URL: "https://127.0.0.1:1", Name: "ctl-b"}} { + if err := st.PutControlPlane(t.Context(), cp); err != nil { + t.Fatal(err) + } + } + if _, err := st.AcquireLeadership(t.Context(), control.ControlRole, "ctl-a-0123456789abcdef", time.Minute); err != nil { + t.Fatal(err) + } + return c +} + +// ask sends one request through the standby's admin API. +func (c *forwardCluster) ask(t *testing.T, path string) (*http.Response, map[string]string) { + t.Helper() + local := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{"path": "local"}) + }) + srv := httptest.NewServer(c.standby.standby(local)) + defer srv.Close() + req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, srv.URL+path, nil) + req.Header.Set("Authorization", "Bearer the-standbys-own-token") + req.Header.Set("X-Llmgw-Actor", "root@ctl-b") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + var got map[string]string + _ = json.Unmarshal(b, &got) + return resp, got +} + +func TestAStandbysAdminAPIIsTheLeaders(t *testing.T) { + c := newForwardCluster(t) + resp, got := c.ask(t, "/api/v1/summary") + if resp.StatusCode != http.StatusOK || got["path"] != "/api/v1/summary" { + t.Fatalf("%d %v", resp.StatusCode, got) + } + if resp.Header.Get(hdrAnsweredBy) != "ctl-a" || got["by"] != "ctl-b" { + t.Errorf("answered by %q, asked by %q", resp.Header.Get(hdrAnsweredBy), got["by"]) + } + if got["auth"] != "" { + t.Error("the standby's token was passed on to the leader") + } + if got["actor"] != "root@ctl-b" { + t.Errorf("who asked was lost: %q", got["actor"]) + } + // What is not the API stays on the machine: the dashboard, pprof. + if _, got := c.ask(t, "/debug/pprof/"); got["path"] != "local" { + t.Errorf("a non-API path was forwarded: %v", got) + } +} + +func TestTheLeaderAnswersOnlyControlPlanesAndOnlyWhileItLeads(t *testing.T) { + c := newForwardCluster(t) + call := func(cert string) int { + cfg := tlsconf.Config{CA: filepath.Join(c.dir, "ca.pem"), Cert: filepath.Join(c.dir, cert+".pem"), Key: filepath.Join(c.dir, cert+"-key.pem")} + tc, err := cfg.ClientConfig("127.0.0.1") + if err != nil { + t.Fatal(err) + } + tc.NextProtos = nil + client := &http.Client{Transport: &http.Transport{TLSClientConfig: tc}} + resp, err := client.Get(c.leaderURL + forwardPrefix + "/api/v1/summary") + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + return resp.StatusCode + } + if code := call("ctl-b"); code != http.StatusOK { + t.Errorf("a control plane was refused: %d", code) + } + // A node's certificate is from the same authority, and is no way in. + if code := call("node1"); code != http.StatusForbidden { + t.Errorf("a node reached the admin API: %d", code) + } + c.leaderLeads.v.Store(false) + if code := call("ctl-b"); code != http.StatusServiceUnavailable { + t.Errorf("a control plane that no longer leads answered: %d", code) + } +} + +func TestAStandbyWithNoLeaderSaysSo(t *testing.T) { + c := newForwardCluster(t) + if err := c.st.ReleaseLeadership(t.Context(), control.ControlRole, "ctl-a-0123456789abcdef"); err != nil { + t.Fatal(err) + } + resp, got := c.ask(t, "/api/v1/summary") + if resp.StatusCode != http.StatusServiceUnavailable || !strings.Contains(got["error"], "standing by") { + t.Errorf("%d %v", resp.StatusCode, got) + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 342bffd..173810c 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -502,8 +502,20 @@ func newServerCmd() *cobra.Command { if err != nil { return err } + adminHandler := auditAPI(rt, adminMux, log) + // With a rival, a standby's admin API is the leader's: what the + // console and ctl show lives in the leader's memory. + var forward *leaderForward + if admin != nil && admin.Leadership() != nil && rt.Store() != nil && cfg.AgentTLS.Enabled() { + self, _ := os.Hostname() + forward = &leaderForward{st: rt.Store(), lead: admin.Leadership(), self: self, tls: cfg.AgentTLS, log: log} + } + served := adminHandler + if forward != nil { + served = forward.standby(adminHandler) + } adminSrv := withLimits(&http.Server{ - Handler: router.AdminGuard(router.AdminAuth(adminToken, cfg.DebugPprof, auditAPI(rt, adminMux, log))), + Handler: router.AdminGuard(router.AdminAuth(adminToken, cfg.DebugPprof, served)), ReadTimeout: time.Minute, }) @@ -569,6 +581,9 @@ func newServerCmd() *cobra.Command { if agentHandler != nil { mux := http.NewServeMux() mux.Handle(agentPath, agentHandler) + if forward != nil { + mux.Handle(forwardPrefix+"/", router.AdminGuard(forward.leaderSide(adminHandler))) + } // No read timeout: an agent's stream is one request that // lasts as long as the agent does. agentSrv = withLimits(&http.Server{Handler: mux}) diff --git a/internal/cli/serveradd.go b/internal/cli/serveradd.go index 5bae60f..d95bbd1 100644 --- a/internal/cli/serveradd.go +++ b/internal/cli/serveradd.go @@ -121,6 +121,7 @@ func (sa *serverAdd) run(ctx context.Context) error { {"connect", sa.connect}, {"binary", sa.binary}, {"directories", sa.dirs}, + {"this certificate", sa.ownCertificate}, {"certificate", sa.certificate}, {"configuration", sa.configuration}, {"store", sa.storeAccess}, @@ -231,6 +232,40 @@ sudo install -d -o llmgw -g llmgw -m 0750 /var/lib/llmgw`, nil) return err } +// ownCertificate makes this control plane's certificate one it can also +// present as a client — a standby asks the leader for the console's data +// with it — reissuing one from before that with the same names. The running +// server reads it as it starts. +func (sa *serverAdd) ownCertificate(context.Context) error { + path := filepath.Join(sa.etcDir, "server.pem") + cert, err := readCert(path) + if err != nil { + return err + } + if clientCapable(cert) { + sa.kept("this control plane's certificate") + return nil + } + ca, err := loadCA(sa.pkiDir) + if err != nil { + return fmt.Errorf("the authority's key is not in %s: run this where it is (%v)", sa.pkiDir, err) + } + hosts := strings.Split(sanNames(cert), ", ") + leaf, err := ca.IssuePeer(cert.Subject.CommonName, hosts) + if err != nil { + return err + } + if err := os.WriteFile(path, leaf.CertPEM, 0o644); err != nil { //nolint:gosec // a certificate + return err + } + if err := os.WriteFile(filepath.Join(sa.etcDir, "server-key.pem"), leaf.KeyPEM, 0o640); err != nil { + return err + } + sa.done("this control plane's certificate reissued for %s, to ask the leader as well as answer;\n"+ + " restart llmgw-server here for it to take effect — %s takes over meanwhile", strings.Join(hosts, ", "), sa.name) + return nil +} + // certificate issues it a server certificate from this cluster's authority, // for its name and addresses, unless it holds one that already names them. func (sa *serverAdd) certificate(ctx context.Context) error { @@ -255,14 +290,14 @@ func (sa *serverAdd) certificate(ctx context.Context) error { for _, w := range want { missing = missing || !slices.Contains(names, w) } - if !missing { + if !missing && clientCapable(cert) { sa.kept("certificate for %s", strings.Join(names, ", ")) return sa.put(ctx, "/etc/llmgw/ca.pem", "0644", "root:root", ca.PEM) } } } } - leaf, err := ca.IssueServer(sa.name, want) + leaf, err := ca.IssuePeer(sa.name, want) if err != nil { return err } @@ -545,3 +580,9 @@ vrrp_instance llmgw { } `, iface, priority, pass, vip, prefix) } + +// clientCapable says a control plane's certificate can also present itself +// as a client, as a standby does to the leader. +func clientCapable(cert *x509.Certificate) bool { + return slices.Contains(cert.ExtKeyUsage, x509.ExtKeyUsageClientAuth) +} diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index de7015a..d797344 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -462,7 +462,10 @@ func (in *serverInit) ensureCerts(context.Context) error { // A certificate from this authority naming every address is left // alone. One naming fewer is reissued with the union, so an address // that worked yesterday is never dropped by adding one today. - if len(missing) == 0 && cert.CheckSignatureFrom(ca.Cert) == nil { + // And one that can also present itself as a client, which a + // standby's admin API needs to reach the leader's; certificates + // from before that are reissued. + if len(missing) == 0 && cert.CheckSignatureFrom(ca.Cert) == nil && clientCapable(cert) { in.kept("server certificate for %s", strings.Join(have, ", ")) return nil } @@ -477,7 +480,7 @@ func (in *serverInit) ensureCerts(context.Context) error { if name == "" { name = "llmgw-control" } - leaf, err := ca.IssueServer(name, want) + leaf, err := ca.IssuePeer(name, want) if err != nil { return err } diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index 0ac4599..3f15914 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -1065,7 +1065,7 @@ func (m consoleModel) panel() (title, body, keys string) { c := &cobra.Command{} c.SetOut(&b) printSummary(c, m.data.sum) - return "Overview", "\n" + indent(b.String(), " "), keys + return "Overview", m.standbyNote() + "\n" + indent(b.String(), " "), keys case cNodes, cLogs: if m.detail && m.row < len(m.data.nodes) { n := m.data.nodes[m.row] @@ -1569,3 +1569,21 @@ func runConsole(ctx context.Context, out io.Writer, file string) error { const floatingQuestion = "A floating address for clients, kept on whichever control plane leads,\n" + "as 10.0.0.161/24 — a free one on this network. Empty for none, when a load\n" + "balancer in front checks /ready instead." + +// standbyNote says, on a control plane standing by, whose cluster the +// console shows: the leader's, which this one's admin API passes it on to. +func (m consoleModel) standbyNote() string { + var here, leader string + for _, cp := range m.data.sum.ControlPlanes.Members { + if cp.This { + here = cp.State + } + if cp.State == "leading" { + leader = cp.Name + } + } + if here != "standing by" || leader == "" { + return "" + } + return "\n " + sDim.Render("Standing by. Shown, and changed, on the leader: "+leader) + "\n" +} diff --git a/internal/cli/tui_setup_test.go b/internal/cli/tui_setup_test.go index a4d8c7f..fc0b4e5 100644 --- a/internal/cli/tui_setup_test.go +++ b/internal/cli/tui_setup_test.go @@ -203,3 +203,18 @@ func TestTheConsoleAddsAControlPlaneWithAFloatingAddress(t *testing.T) { t.Errorf("dialog %q: %s", c.dialog, c.dialogText) } } + +func TestTheConsoleOnAStandbySaysWhoseClusterItShows(t *testing.T) { + m := consoleModel{menu: cOverview, loaded: true, width: 120, height: 40} + m.data.sum.ControlPlanes = adminapi.ControlPlanes{Members: []adminapi.ControlPlane{ + {Name: "llmgw-ctl", State: "leading"}, + {Name: "llmgw-ctl02", State: "standing by", This: true}, + }} + if v := m.View(); !strings.Contains(v, "Standing by. Shown, and changed, on the leader: llmgw-ctl") { + t.Errorf("the overview on a standby:\n%s", v) + } + m.data.sum.ControlPlanes.Members[0].This, m.data.sum.ControlPlanes.Members[1].This = true, false + if v := m.View(); strings.Contains(v, "Standing by.") { + t.Error("the leader's console says it is standing by") + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 0b50dd5..f95235c 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -1162,8 +1162,18 @@ func (a *API) removeModel(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, Removed{Removed: name}) } -func (a *API) summary(w http.ResponseWriter, _ *http.Request) { - writeJSON(w, http.StatusOK, a.Summary()) +func (a *API) summary(w http.ResponseWriter, r *http.Request) { + s := a.Summary() + // Asked through a standby, which passes on the console's questions to + // the leader: "this one" is the standby the console is on. + if by := r.Header.Get("X-Llmgw-Forwarded-By"); by != "" && len(s.ControlPlanes.Members) > 0 { + members := slices.Clone(s.ControlPlanes.Members) + for i := range members { + members[i].This = members[i].Name == by + } + s.ControlPlanes.Members = members + } + writeJSON(w, http.StatusOK, s) } func (a *API) explainPlacement(w http.ResponseWriter, r *http.Request) { diff --git a/internal/control/control.go b/internal/control/control.go index bf13c85..7630919 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -231,6 +231,10 @@ func (c *Controller) SetLeadership(l *Leadership) { c.rec.SetLeadership(l) } +// Leadership is what SetLeadership set; nil when there is nobody to compete +// with. +func (c *Controller) Leadership() *Leadership { return c.lead } + // IsLeader reports whether this process is placing work. True when leadership // was never configured, because then there is nobody to compete with. func (c *Controller) IsLeader() bool { return c.lead == nil || c.lead.IsLeader() } diff --git a/internal/control/leadership.go b/internal/control/leadership.go index 580d3df..493a6a6 100644 --- a/internal/control/leadership.go +++ b/internal/control/leadership.go @@ -7,6 +7,7 @@ import ( "errors" "log/slog" "os" + "strings" "sync/atomic" "time" @@ -183,3 +184,16 @@ func newOwnerID() string { } return host + "-" + hex.EncodeToString(b[:]) } + +// ControlRole is the lease's name in the store, for readers that ask who +// holds it without competing. +const ControlRole = controlRole + +// HolderName is the machine an owner id names: the hostname it was made +// from, without the random part. +func HolderName(owner string) string { + if i := strings.LastIndexByte(owner, '-'); i > 0 { + return owner[:i] + } + return owner +}