A standby's portal is the leader's; the changelog as the release notes read it
A proxy left pointing at a standby showed an empty cluster: the portal and the admin pages read the standby's own memory. A standby now passes its client port to the leader's, all but /health, /ready and /leader, which must describe the machine a load balancer is choosing; and its admin port passes the dashboard as well as the API. The leader trusts the control planes' X-Forwarded-For, so clients keep their own address. The changelog's candidate section is "Unreleased" again: release notes for an rc read that section, and v0.11.3-rc.1 got none. The e2e wait for two endpoints is 30s: under the race detector on a busy runner, 15s was missed once. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c5b34978f5
commit
abaf086fd5
7 files changed
+172
-23
No files matched your search
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
@@ -16,6 +17,7 @@ import (
|
||||
|
||||
"golang.org/x/net/http2"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/clientip"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
@@ -53,7 +55,7 @@ type leaderForward struct {
|
||||
// process is not it.
|
||||
func (f *leaderForward) standby(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.HasPrefix(r.URL.Path, "/api/") || f.lead.IsLeader() {
|
||||
if adminStaysHere(r.URL.Path) || f.lead.IsLeader() {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
@@ -90,6 +92,89 @@ func (f *leaderForward) standby(next http.Handler) http.Handler {
|
||||
})
|
||||
}
|
||||
|
||||
// adminStaysHere are the admin paths about this process rather than the
|
||||
// cluster: its profile, its own metrics, its own health. Never forwarded, and
|
||||
// never answered for another — pprof dumps the heap, keys and all.
|
||||
func adminStaysHere(path string) bool {
|
||||
return strings.HasPrefix(path, "/debug/") || path == "/metrics" || path == "/health"
|
||||
}
|
||||
|
||||
// publicStaysHere are the client port's paths that must describe this
|
||||
// process: a load balancer and keepalived ask them which machine to send
|
||||
// traffic to, and a standby saying the leader is ready would draw it.
|
||||
func publicStaysHere(path string) bool {
|
||||
return path == "/health" || path == "/ready" || path == "/leader" || path == "/metrics"
|
||||
}
|
||||
|
||||
// public wraps the client-facing handler: on a standby, everything but its
|
||||
// own health goes to the leader's client port — the portal, its admin pages
|
||||
// and the API, which on a standby would show an empty cluster and serve
|
||||
// nothing. Plain HTTP, as the port is; the leader trusts a control plane's
|
||||
// X-Forwarded-For, so a client is still known by its own address.
|
||||
func (f *leaderForward) public(next http.Handler, port string) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Once only: a request another control plane forwarded is answered
|
||||
// here, leading or not, rather than sent on while the lease moves.
|
||||
if publicStaysHere(r.URL.Path) || f.lead.IsLeader() || r.Header.Get(hdrForwardedBy) != "" {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
target, _, err := f.leader(r.Context())
|
||||
if err != nil || target.Hostname() == "" {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
host := net.JoinHostPort(target.Hostname(), port)
|
||||
p := &httputil.ReverseProxy{
|
||||
Rewrite: func(pr *httputil.ProxyRequest) {
|
||||
pr.Out.URL.Scheme, pr.Out.URL.Host = "http", host
|
||||
// The name the client used, for the portal's links and cookies.
|
||||
pr.Out.Host = pr.In.Host
|
||||
peer, _, _ := net.SplitHostPort(pr.In.RemoteAddr)
|
||||
xff := pr.In.Header.Get("X-Forwarded-For")
|
||||
if xff != "" {
|
||||
xff += ", "
|
||||
}
|
||||
pr.Out.Header.Set("X-Forwarded-For", xff+peer)
|
||||
for _, h := range []string{"X-Forwarded-Proto", "X-Forwarded-Host"} {
|
||||
if v := pr.In.Header.Get(h); v != "" {
|
||||
pr.Out.Header.Set(h, v)
|
||||
}
|
||||
}
|
||||
pr.Out.Header.Set(hdrForwardedBy, f.self)
|
||||
},
|
||||
FlushInterval: -1,
|
||||
ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
|
||||
writeForwardError(w, http.StatusBadGateway, "this control plane is standing by, and the leader did not answer: "+err.Error())
|
||||
},
|
||||
}
|
||||
p.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// trustControlPlanes keeps the client port trusting the other control
|
||||
// planes' X-Forwarded-For, besides router.trusted_proxies: a standby passes
|
||||
// requests on, and the client is the address it was given, not the standby.
|
||||
// Read again every half minute, for one added since.
|
||||
func (f *leaderForward) trustControlPlanes(ctx context.Context, configured []string) {
|
||||
for {
|
||||
list := slices.Clone(configured)
|
||||
if cps, err := f.st.ControlPlanes(ctx); err == nil {
|
||||
for _, cp := range cps {
|
||||
if u, err := url.Parse(cp.URL); err == nil && u.Hostname() != "" {
|
||||
list = append(list, u.Hostname())
|
||||
}
|
||||
}
|
||||
}
|
||||
clientip.SetTrusted(list)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(30 * time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// leader is where the lease's holder is reached, and its name.
|
||||
func (f *leaderForward) leader(ctx context.Context) (*url.URL, string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
@@ -99,6 +184,10 @@ func (f *leaderForward) leader(ctx context.Context) (*url.URL, string, error) {
|
||||
return nil, "", errors.New("no control plane holds the lease right now")
|
||||
}
|
||||
name := control.HolderName(lease.Owner)
|
||||
if name == f.self {
|
||||
// This machine's previous process, whose lease has not run out.
|
||||
return nil, name, errors.New("the lease is still held by this machine's previous run; it runs out within seconds")
|
||||
}
|
||||
cps, err := f.st.ControlPlanes(ctx)
|
||||
if err != nil {
|
||||
return nil, name, err
|
||||
@@ -117,7 +206,7 @@ func (f *leaderForward) leader(ctx context.Context) (*url.URL, string, error) {
|
||||
func (f *leaderForward) leaderSide(admin http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path := strings.TrimPrefix(r.URL.Path, forwardPrefix)
|
||||
if !strings.HasPrefix(path, "/api/") {
|
||||
if adminStaysHere(path) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user