A standby's admin API is the leader's

The console and ctl on a standby showed its own empty view: nodes and
replicas live in the leader's memory. A standby now passes /api/ to the
leader over the agent port, with the cluster's certificates both ways; the
leader answers only a certificate naming a control plane the store lists,
and only while it leads. Control plane certificates are issued for client
use as well, and server add reissues older ones.

An agent's reconnect backoff starts again after a session that worked, so
each failover it lives through no longer makes the next one slower.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 02:27:11 +02:00
1 parent adef25d709
commit 1d1409faad
12 files changed
+510 -13

No files matched your search

+168
View File
@@ -0,0 +1,168 @@
package cli
import (
"context"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net/http"
"net/http/httputil"
"net/url"
"slices"
"strings"
"time"
"golang.org/x/net/http2"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
// forwardPrefix is where the leader answers a standby's admin API, on the
// agent port: the one port control planes already reach each other on, with
// the cluster's certificates both ways. The admin port itself listens on
// loopback, and should — it serves pprof.
const forwardPrefix = "/_admin"
// Headers a forwarded request and its answer carry: which standby asked, and
// which control plane answered.
const (
hdrForwardedBy = "X-Llmgw-Forwarded-By"
hdrAnsweredBy = "X-Llmgw-Answered-By"
)
// leaderForward sends a standby's admin API to the leader.
//
// Everything the console and ctl show is the leader's: nodes, replicas and
// jobs live in its memory, put there by the agent streams only it holds. A
// standby answering from its own would show an empty cluster, and accept
// changes it cannot carry out. So a standby answers nothing under /api/
// itself; the leader does, and the console looks the same on either machine.
type leaderForward struct {
st *store.DB
lead interface{ IsLeader() bool }
self string
tls tlsconf.Config
log *slog.Logger
}
// standby wraps the admin handler: /api/ goes to the leader while this
// process is not it.
func (f *leaderForward) standby(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.URL.Path, "/api/") || f.lead.IsLeader() {
next.ServeHTTP(w, r)
return
}
target, name, err := f.leader(r.Context())
if err != nil {
writeForwardError(w, http.StatusServiceUnavailable,
"this control plane is standing by, and cannot reach the leader: "+err.Error())
return
}
tc, err := f.tls.ClientConfig(target.Hostname())
if err != nil {
writeForwardError(w, http.StatusInternalServerError, err.Error())
return
}
p := &httputil.ReverseProxy{
Rewrite: func(pr *httputil.ProxyRequest) {
pr.Out.URL.Scheme, pr.Out.URL.Host = "https", target.Host
pr.Out.URL.Path = forwardPrefix + pr.In.URL.Path
pr.Out.URL.RawPath = ""
pr.Out.Host = target.Host
// The leader's own token is not this one's, and the
// certificate is what it believes.
pr.Out.Header.Del("Authorization")
pr.Out.Header.Set(hdrForwardedBy, f.self)
},
Transport: &http2.Transport{TLSClientConfig: tc},
// Streams, such as a job's progress, pass through as they come.
FlushInterval: -1,
ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
writeForwardError(w, http.StatusBadGateway, fmt.Sprintf("the leader, %s, did not answer: %v", name, err))
},
}
p.ServeHTTP(w, r)
})
}
// leader is where the lease's holder is reached, and its name.
func (f *leaderForward) leader(ctx context.Context) (*url.URL, string, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
lease, err := f.st.Leadership(ctx, control.ControlRole)
if err != nil || lease.Owner == "" || time.Now().After(lease.ExpiresAt) {
return nil, "", errors.New("no control plane holds the lease right now")
}
name := control.HolderName(lease.Owner)
cps, err := f.st.ControlPlanes(ctx)
if err != nil {
return nil, name, err
}
for _, cp := range cps {
if cp.Name == name {
u, err := url.Parse(cp.URL)
return u, name, err
}
}
return nil, name, fmt.Errorf("%s leads, and is not among the control planes the store lists", name)
}
// leaderSide answers forwarded requests, on the agent port: only from another
// control plane of this cluster, only while this one leads, and only the API.
func (f *leaderForward) leaderSide(admin http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
path := strings.TrimPrefix(r.URL.Path, forwardPrefix)
if !strings.HasPrefix(path, "/api/") {
http.NotFound(w, r)
return
}
if err := f.fromControlPlane(r); err != nil {
writeForwardError(w, http.StatusForbidden, err.Error())
return
}
if !f.lead.IsLeader() {
// Not passed on again: two standbys must not bounce a request
// between them while the lease moves.
writeForwardError(w, http.StatusServiceUnavailable, "this control plane no longer leads; ask again")
return
}
r2 := r.Clone(r.Context())
r2.URL.Path, r2.URL.RawPath = path, ""
w.Header().Set(hdrAnsweredBy, f.self)
admin.ServeHTTP(w, r2)
})
}
// fromControlPlane checks the client certificate names a control plane the
// store lists. A node's certificate is from the same authority, and a node
// must not reach the admin API by presenting it.
func (f *leaderForward) fromControlPlane(r *http.Request) error {
if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 {
return errors.New("a control plane's certificate is required")
}
cert := r.TLS.VerifiedChains[0][0]
if !slices.Contains(cert.ExtKeyUsage, x509.ExtKeyUsageServerAuth) {
return errors.New("not a control plane's certificate")
}
cps, err := f.st.ControlPlanes(r.Context())
if err != nil {
return err
}
for _, cp := range cps {
if cp.Name == cert.Subject.CommonName {
return nil
}
}
return fmt.Errorf("%s is not a control plane of this cluster", cert.Subject.CommonName)
}
func writeForwardError(w http.ResponseWriter, code int, msg string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
}
+188
View File
@@ -0,0 +1,188 @@
package cli
import (
"encoding/json"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
type leading struct{ v atomic.Bool }
func (l *leading) IsLeader() bool { return l.v.Load() }
// forwardCluster is two control planes of one cluster on one store: a
// leader whose agent port answers forwarded admin requests, and a standby.
type forwardCluster struct {
st *store.DB
dir string
leaderLeads *leading
standby, leader *leaderForward
leaderURL string
}
func newForwardCluster(t *testing.T) *forwardCluster {
t.Helper()
dir := t.TempDir()
ca, err := tlsconf.NewCA("test CA")
if err != nil {
t.Fatal(err)
}
if err := ca.Write(dir); err != nil {
t.Fatal(err)
}
for _, name := range []string{"ctl-a", "ctl-b"} {
leaf, err := ca.IssuePeer(name, []string{"127.0.0.1"})
if err != nil {
t.Fatal(err)
}
if err := leaf.Write(dir, name); err != nil {
t.Fatal(err)
}
}
node, err := ca.IssueClient("node1")
if err != nil {
t.Fatal(err)
}
if err := node.Write(dir, "node1"); err != nil {
t.Fatal(err)
}
cfgFor := func(name string) tlsconf.Config {
return tlsconf.Config{CA: filepath.Join(dir, "ca.pem"), Cert: filepath.Join(dir, name+".pem"), Key: filepath.Join(dir, name+"-key.pem")}
}
st, err := store.Open(t.Context(), filepath.Join(dir, "s.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { st.Close() })
c := &forwardCluster{st: st, dir: dir, leaderLeads: &leading{}}
c.leaderLeads.v.Store(true)
log := slog.New(slog.DiscardHandler)
c.leader = &leaderForward{st: st, lead: c.leaderLeads, self: "ctl-a", tls: cfgFor("ctl-a"), log: log}
standbyLeads := &leading{}
c.standby = &leaderForward{st: st, lead: standbyLeads, self: "ctl-b", tls: cfgFor("ctl-b"), log: log}
// The leader's admin API: says whose it is, and who it was asked by.
admin := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]string{"path": r.URL.Path, "by": r.Header.Get(hdrForwardedBy),
"auth": r.Header.Get("Authorization"), "actor": r.Header.Get("X-Llmgw-Actor")})
})
mux := http.NewServeMux()
mux.Handle(forwardPrefix+"/", c.leader.leaderSide(admin))
srv := httptest.NewUnstartedServer(mux)
tc, err := cfgFor("ctl-a").ServerConfig()
if err != nil {
t.Fatal(err)
}
srv.TLS = tc
srv.EnableHTTP2 = true
srv.StartTLS()
t.Cleanup(srv.Close)
c.leaderURL = srv.URL
for _, cp := range []store.ControlPlane{{URL: srv.URL, Name: "ctl-a"}, {URL: "https://127.0.0.1:1", Name: "ctl-b"}} {
if err := st.PutControlPlane(t.Context(), cp); err != nil {
t.Fatal(err)
}
}
if _, err := st.AcquireLeadership(t.Context(), control.ControlRole, "ctl-a-0123456789abcdef", time.Minute); err != nil {
t.Fatal(err)
}
return c
}
// ask sends one request through the standby's admin API.
func (c *forwardCluster) ask(t *testing.T, path string) (*http.Response, map[string]string) {
t.Helper()
local := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]string{"path": "local"})
})
srv := httptest.NewServer(c.standby.standby(local))
defer srv.Close()
req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, srv.URL+path, nil)
req.Header.Set("Authorization", "Bearer the-standbys-own-token")
req.Header.Set("X-Llmgw-Actor", "root@ctl-b")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
b, _ := io.ReadAll(resp.Body)
var got map[string]string
_ = json.Unmarshal(b, &got)
return resp, got
}
func TestAStandbysAdminAPIIsTheLeaders(t *testing.T) {
c := newForwardCluster(t)
resp, got := c.ask(t, "/api/v1/summary")
if resp.StatusCode != http.StatusOK || got["path"] != "/api/v1/summary" {
t.Fatalf("%d %v", resp.StatusCode, got)
}
if resp.Header.Get(hdrAnsweredBy) != "ctl-a" || got["by"] != "ctl-b" {
t.Errorf("answered by %q, asked by %q", resp.Header.Get(hdrAnsweredBy), got["by"])
}
if got["auth"] != "" {
t.Error("the standby's token was passed on to the leader")
}
if got["actor"] != "root@ctl-b" {
t.Errorf("who asked was lost: %q", got["actor"])
}
// What is not the API stays on the machine: the dashboard, pprof.
if _, got := c.ask(t, "/debug/pprof/"); got["path"] != "local" {
t.Errorf("a non-API path was forwarded: %v", got)
}
}
func TestTheLeaderAnswersOnlyControlPlanesAndOnlyWhileItLeads(t *testing.T) {
c := newForwardCluster(t)
call := func(cert string) int {
cfg := tlsconf.Config{CA: filepath.Join(c.dir, "ca.pem"), Cert: filepath.Join(c.dir, cert+".pem"), Key: filepath.Join(c.dir, cert+"-key.pem")}
tc, err := cfg.ClientConfig("127.0.0.1")
if err != nil {
t.Fatal(err)
}
tc.NextProtos = nil
client := &http.Client{Transport: &http.Transport{TLSClientConfig: tc}}
resp, err := client.Get(c.leaderURL + forwardPrefix + "/api/v1/summary")
if err != nil {
t.Fatal(err)
}
resp.Body.Close()
return resp.StatusCode
}
if code := call("ctl-b"); code != http.StatusOK {
t.Errorf("a control plane was refused: %d", code)
}
// A node's certificate is from the same authority, and is no way in.
if code := call("node1"); code != http.StatusForbidden {
t.Errorf("a node reached the admin API: %d", code)
}
c.leaderLeads.v.Store(false)
if code := call("ctl-b"); code != http.StatusServiceUnavailable {
t.Errorf("a control plane that no longer leads answered: %d", code)
}
}
func TestAStandbyWithNoLeaderSaysSo(t *testing.T) {
c := newForwardCluster(t)
if err := c.st.ReleaseLeadership(t.Context(), control.ControlRole, "ctl-a-0123456789abcdef"); err != nil {
t.Fatal(err)
}
resp, got := c.ask(t, "/api/v1/summary")
if resp.StatusCode != http.StatusServiceUnavailable || !strings.Contains(got["error"], "standing by") {
t.Errorf("%d %v", resp.StatusCode, got)
}
}
+16 -1
View File
@@ -502,8 +502,20 @@ func newServerCmd() *cobra.Command {
if err != nil {
return err
}
adminHandler := auditAPI(rt, adminMux, log)
// With a rival, a standby's admin API is the leader's: what the
// console and ctl show lives in the leader's memory.
var forward *leaderForward
if admin != nil && admin.Leadership() != nil && rt.Store() != nil && cfg.AgentTLS.Enabled() {
self, _ := os.Hostname()
forward = &leaderForward{st: rt.Store(), lead: admin.Leadership(), self: self, tls: cfg.AgentTLS, log: log}
}
served := adminHandler
if forward != nil {
served = forward.standby(adminHandler)
}
adminSrv := withLimits(&http.Server{
Handler: router.AdminGuard(router.AdminAuth(adminToken, cfg.DebugPprof, auditAPI(rt, adminMux, log))),
Handler: router.AdminGuard(router.AdminAuth(adminToken, cfg.DebugPprof, served)),
ReadTimeout: time.Minute,
})
@@ -569,6 +581,9 @@ func newServerCmd() *cobra.Command {
if agentHandler != nil {
mux := http.NewServeMux()
mux.Handle(agentPath, agentHandler)
if forward != nil {
mux.Handle(forwardPrefix+"/", router.AdminGuard(forward.leaderSide(adminHandler)))
}
// No read timeout: an agent's stream is one request that
// lasts as long as the agent does.
agentSrv = withLimits(&http.Server{Handler: mux})
+43 -2
View File
@@ -121,6 +121,7 @@ func (sa *serverAdd) run(ctx context.Context) error {
{"connect", sa.connect},
{"binary", sa.binary},
{"directories", sa.dirs},
{"this certificate", sa.ownCertificate},
{"certificate", sa.certificate},
{"configuration", sa.configuration},
{"store", sa.storeAccess},
@@ -231,6 +232,40 @@ sudo install -d -o llmgw -g llmgw -m 0750 /var/lib/llmgw`, nil)
return err
}
// ownCertificate makes this control plane's certificate one it can also
// present as a client — a standby asks the leader for the console's data
// with it — reissuing one from before that with the same names. The running
// server reads it as it starts.
func (sa *serverAdd) ownCertificate(context.Context) error {
path := filepath.Join(sa.etcDir, "server.pem")
cert, err := readCert(path)
if err != nil {
return err
}
if clientCapable(cert) {
sa.kept("this control plane's certificate")
return nil
}
ca, err := loadCA(sa.pkiDir)
if err != nil {
return fmt.Errorf("the authority's key is not in %s: run this where it is (%v)", sa.pkiDir, err)
}
hosts := strings.Split(sanNames(cert), ", ")
leaf, err := ca.IssuePeer(cert.Subject.CommonName, hosts)
if err != nil {
return err
}
if err := os.WriteFile(path, leaf.CertPEM, 0o644); err != nil { //nolint:gosec // a certificate
return err
}
if err := os.WriteFile(filepath.Join(sa.etcDir, "server-key.pem"), leaf.KeyPEM, 0o640); err != nil {
return err
}
sa.done("this control plane's certificate reissued for %s, to ask the leader as well as answer;\n"+
" restart llmgw-server here for it to take effect — %s takes over meanwhile", strings.Join(hosts, ", "), sa.name)
return nil
}
// certificate issues it a server certificate from this cluster's authority,
// for its name and addresses, unless it holds one that already names them.
func (sa *serverAdd) certificate(ctx context.Context) error {
@@ -255,14 +290,14 @@ func (sa *serverAdd) certificate(ctx context.Context) error {
for _, w := range want {
missing = missing || !slices.Contains(names, w)
}
if !missing {
if !missing && clientCapable(cert) {
sa.kept("certificate for %s", strings.Join(names, ", "))
return sa.put(ctx, "/etc/llmgw/ca.pem", "0644", "root:root", ca.PEM)
}
}
}
}
leaf, err := ca.IssueServer(sa.name, want)
leaf, err := ca.IssuePeer(sa.name, want)
if err != nil {
return err
}
@@ -545,3 +580,9 @@ vrrp_instance llmgw {
}
`, iface, priority, pass, vip, prefix)
}
// clientCapable says a control plane's certificate can also present itself
// as a client, as a standby does to the leader.
func clientCapable(cert *x509.Certificate) bool {
return slices.Contains(cert.ExtKeyUsage, x509.ExtKeyUsageClientAuth)
}
+5 -2
View File
@@ -462,7 +462,10 @@ func (in *serverInit) ensureCerts(context.Context) error {
// A certificate from this authority naming every address is left
// alone. One naming fewer is reissued with the union, so an address
// that worked yesterday is never dropped by adding one today.
if len(missing) == 0 && cert.CheckSignatureFrom(ca.Cert) == nil {
// And one that can also present itself as a client, which a
// standby's admin API needs to reach the leader's; certificates
// from before that are reissued.
if len(missing) == 0 && cert.CheckSignatureFrom(ca.Cert) == nil && clientCapable(cert) {
in.kept("server certificate for %s", strings.Join(have, ", "))
return nil
}
@@ -477,7 +480,7 @@ func (in *serverInit) ensureCerts(context.Context) error {
if name == "" {
name = "llmgw-control"
}
leaf, err := ca.IssueServer(name, want)
leaf, err := ca.IssuePeer(name, want)
if err != nil {
return err
}
+19 -1
View File
@@ -1065,7 +1065,7 @@ func (m consoleModel) panel() (title, body, keys string) {
c := &cobra.Command{}
c.SetOut(&b)
printSummary(c, m.data.sum)
return "Overview", "\n" + indent(b.String(), " "), keys
return "Overview", m.standbyNote() + "\n" + indent(b.String(), " "), keys
case cNodes, cLogs:
if m.detail && m.row < len(m.data.nodes) {
n := m.data.nodes[m.row]
@@ -1569,3 +1569,21 @@ func runConsole(ctx context.Context, out io.Writer, file string) error {
const floatingQuestion = "A floating address for clients, kept on whichever control plane leads,\n" +
"as 10.0.0.161/24 — a free one on this network. Empty for none, when a load\n" +
"balancer in front checks /ready instead."
// standbyNote says, on a control plane standing by, whose cluster the
// console shows: the leader's, which this one's admin API passes it on to.
func (m consoleModel) standbyNote() string {
var here, leader string
for _, cp := range m.data.sum.ControlPlanes.Members {
if cp.This {
here = cp.State
}
if cp.State == "leading" {
leader = cp.Name
}
}
if here != "standing by" || leader == "" {
return ""
}
return "\n " + sDim.Render("Standing by. Shown, and changed, on the leader: "+leader) + "\n"
}
+15
View File
@@ -203,3 +203,18 @@ func TestTheConsoleAddsAControlPlaneWithAFloatingAddress(t *testing.T) {
t.Errorf("dialog %q: %s", c.dialog, c.dialogText)
}
}
func TestTheConsoleOnAStandbySaysWhoseClusterItShows(t *testing.T) {
m := consoleModel{menu: cOverview, loaded: true, width: 120, height: 40}
m.data.sum.ControlPlanes = adminapi.ControlPlanes{Members: []adminapi.ControlPlane{
{Name: "llmgw-ctl", State: "leading"},
{Name: "llmgw-ctl02", State: "standing by", This: true},
}}
if v := m.View(); !strings.Contains(v, "Standing by. Shown, and changed, on the leader: llmgw-ctl") {
t.Errorf("the overview on a standby:\n%s", v)
}
m.data.sum.ControlPlanes.Members[0].This, m.data.sum.ControlPlanes.Members[1].This = true, false
if v := m.View(); strings.Contains(v, "Standing by.") {
t.Error("the leader's console says it is standing by")
}
}