A standby's portal is the leader's; the changelog as the release notes read it

A proxy left pointing at a standby showed an empty cluster: the portal and
the admin pages read the standby's own memory. A standby now passes its
client port to the leader's, all but /health, /ready and /leader, which
must describe the machine a load balancer is choosing; and its admin port
passes the dashboard as well as the API. The leader trusts the control
planes' X-Forwarded-For, so clients keep their own address.

The changelog's candidate section is "Unreleased" again: release notes
for an rc read that section, and v0.11.3-rc.1 got none.

The e2e wait for two endpoints is 30s: under the race detector on a busy
runner, 15s was missed once.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 06:34:57 +02:00
1 parent c5b34978f5
commit abaf086fd5
7 files changed
+172 -23

No files matched your search

+91 -2
View File
@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"log/slog"
"net"
"net/http"
"net/http/httputil"
"net/url"
@@ -16,6 +17,7 @@ import (
"golang.org/x/net/http2"
"gitea.mynodes.xyz/ilker/llmgw/internal/clientip"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
@@ -53,7 +55,7 @@ type leaderForward struct {
// process is not it.
func (f *leaderForward) standby(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.URL.Path, "/api/") || f.lead.IsLeader() {
if adminStaysHere(r.URL.Path) || f.lead.IsLeader() {
next.ServeHTTP(w, r)
return
}
@@ -90,6 +92,89 @@ func (f *leaderForward) standby(next http.Handler) http.Handler {
})
}
// adminStaysHere are the admin paths about this process rather than the
// cluster: its profile, its own metrics, its own health. Never forwarded, and
// never answered for another — pprof dumps the heap, keys and all.
func adminStaysHere(path string) bool {
return strings.HasPrefix(path, "/debug/") || path == "/metrics" || path == "/health"
}
// publicStaysHere are the client port's paths that must describe this
// process: a load balancer and keepalived ask them which machine to send
// traffic to, and a standby saying the leader is ready would draw it.
func publicStaysHere(path string) bool {
return path == "/health" || path == "/ready" || path == "/leader" || path == "/metrics"
}
// public wraps the client-facing handler: on a standby, everything but its
// own health goes to the leader's client port — the portal, its admin pages
// and the API, which on a standby would show an empty cluster and serve
// nothing. Plain HTTP, as the port is; the leader trusts a control plane's
// X-Forwarded-For, so a client is still known by its own address.
func (f *leaderForward) public(next http.Handler, port string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Once only: a request another control plane forwarded is answered
// here, leading or not, rather than sent on while the lease moves.
if publicStaysHere(r.URL.Path) || f.lead.IsLeader() || r.Header.Get(hdrForwardedBy) != "" {
next.ServeHTTP(w, r)
return
}
target, _, err := f.leader(r.Context())
if err != nil || target.Hostname() == "" {
next.ServeHTTP(w, r)
return
}
host := net.JoinHostPort(target.Hostname(), port)
p := &httputil.ReverseProxy{
Rewrite: func(pr *httputil.ProxyRequest) {
pr.Out.URL.Scheme, pr.Out.URL.Host = "http", host
// The name the client used, for the portal's links and cookies.
pr.Out.Host = pr.In.Host
peer, _, _ := net.SplitHostPort(pr.In.RemoteAddr)
xff := pr.In.Header.Get("X-Forwarded-For")
if xff != "" {
xff += ", "
}
pr.Out.Header.Set("X-Forwarded-For", xff+peer)
for _, h := range []string{"X-Forwarded-Proto", "X-Forwarded-Host"} {
if v := pr.In.Header.Get(h); v != "" {
pr.Out.Header.Set(h, v)
}
}
pr.Out.Header.Set(hdrForwardedBy, f.self)
},
FlushInterval: -1,
ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
writeForwardError(w, http.StatusBadGateway, "this control plane is standing by, and the leader did not answer: "+err.Error())
},
}
p.ServeHTTP(w, r)
})
}
// trustControlPlanes keeps the client port trusting the other control
// planes' X-Forwarded-For, besides router.trusted_proxies: a standby passes
// requests on, and the client is the address it was given, not the standby.
// Read again every half minute, for one added since.
func (f *leaderForward) trustControlPlanes(ctx context.Context, configured []string) {
for {
list := slices.Clone(configured)
if cps, err := f.st.ControlPlanes(ctx); err == nil {
for _, cp := range cps {
if u, err := url.Parse(cp.URL); err == nil && u.Hostname() != "" {
list = append(list, u.Hostname())
}
}
}
clientip.SetTrusted(list)
select {
case <-ctx.Done():
return
case <-time.After(30 * time.Second):
}
}
}
// leader is where the lease's holder is reached, and its name.
func (f *leaderForward) leader(ctx context.Context) (*url.URL, string, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
@@ -99,6 +184,10 @@ func (f *leaderForward) leader(ctx context.Context) (*url.URL, string, error) {
return nil, "", errors.New("no control plane holds the lease right now")
}
name := control.HolderName(lease.Owner)
if name == f.self {
// This machine's previous process, whose lease has not run out.
return nil, name, errors.New("the lease is still held by this machine's previous run; it runs out within seconds")
}
cps, err := f.st.ControlPlanes(ctx)
if err != nil {
return nil, name, err
@@ -117,7 +206,7 @@ func (f *leaderForward) leader(ctx context.Context) (*url.URL, string, error) {
func (f *leaderForward) leaderSide(admin http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
path := strings.TrimPrefix(r.URL.Path, forwardPrefix)
if !strings.HasPrefix(path, "/api/") {
if adminStaysHere(path) {
http.NotFound(w, r)
return
}