llmgw/internal/cli/leaderforward.go

package cli

import (
	"context"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"log/slog"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"slices"
	"strings"
	"time"

	"golang.org/x/net/http2"

	"gitea.mynodes.xyz/ilker/llmgw/internal/clientip"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control"
	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
	"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)

// forwardPrefix is where the leader answers a standby's admin API, on the
// agent port: the one port control planes already reach each other on, with
// the cluster's certificates both ways. The admin port itself listens on
// loopback, and should — it serves pprof.
const forwardPrefix = "/_admin"

// Headers a forwarded request and its answer carry: which standby asked, and
// which control plane answered.
const (
	hdrForwardedBy = "X-Llmgw-Forwarded-By"
	hdrAnsweredBy  = "X-Llmgw-Answered-By"
)

// leaderForward sends a standby's admin API to the leader.
//
// Everything the console and ctl show is the leader's: nodes, replicas and
// jobs live in its memory, put there by the agent streams only it holds. A
// standby answering from its own would show an empty cluster, and accept
// changes it cannot carry out. So a standby answers nothing under /api/
// itself; the leader does, and the console looks the same on either machine.
type leaderForward struct {
	st   *store.DB
	lead interface{ IsLeader() bool }
	self string
	tls  tlsconf.Config
	log  *slog.Logger
}

// standby wraps the admin handler: /api/ goes to the leader while this
// process is not it.
func (f *leaderForward) standby(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if adminStaysHere(r.URL.Path) || f.lead.IsLeader() {
			next.ServeHTTP(w, r)
			return
		}
		target, name, err := f.leader(r.Context())
		if err != nil {
			writeForwardError(w, http.StatusServiceUnavailable,
				"this control plane is standing by, and cannot reach the leader: "+err.Error())
			return
		}
		tc, err := f.tls.ClientConfig(target.Hostname())
		if err != nil {
			writeForwardError(w, http.StatusInternalServerError, err.Error())
			return
		}
		p := &httputil.ReverseProxy{
			Rewrite: func(pr *httputil.ProxyRequest) {
				pr.Out.URL.Scheme, pr.Out.URL.Host = "https", target.Host
				pr.Out.URL.Path = forwardPrefix + pr.In.URL.Path
				pr.Out.URL.RawPath = ""
				pr.Out.Host = target.Host
				// The leader's own token is not this one's, and the
				// certificate is what it believes.
				pr.Out.Header.Del("Authorization")
				pr.Out.Header.Set(hdrForwardedBy, f.self)
			},
			Transport: &http2.Transport{TLSClientConfig: tc},
			// Streams, such as a job's progress, pass through as they come.
			FlushInterval: -1,
			ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
				writeForwardError(w, http.StatusBadGateway, fmt.Sprintf("the leader, %s, did not answer: %v", name, err))
			},
		}
		p.ServeHTTP(w, r)
	})
}

// adminStaysHere are the admin paths about this process rather than the
// cluster: its profile, its own metrics, its own health. Never forwarded, and
// never answered for another — pprof dumps the heap, keys and all.
func adminStaysHere(path string) bool {
	return strings.HasPrefix(path, "/debug/") || path == "/metrics" || path == "/health"
}

// publicStaysHere are the client port's paths that must describe this
// process: a load balancer and keepalived ask them which machine to send
// traffic to, and a standby saying the leader is ready would draw it.
func publicStaysHere(path string) bool {
	return path == "/health" || path == "/ready" || path == "/leader" || path == "/metrics"
}

// public wraps the client-facing handler: on a standby, everything but its
// own health goes to the leader's client port — the portal, its admin pages
// and the API, which on a standby would show an empty cluster and serve
// nothing. Plain HTTP, as the port is; the leader trusts a control plane's
// X-Forwarded-For, so a client is still known by its own address.
func (f *leaderForward) public(next http.Handler, port string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Once only: a request another control plane forwarded is answered
		// here, leading or not, rather than sent on while the lease moves.
		if publicStaysHere(r.URL.Path) || f.lead.IsLeader() || r.Header.Get(hdrForwardedBy) != "" {
			next.ServeHTTP(w, r)
			return
		}
		target, _, err := f.leader(r.Context())
		if err != nil || target.Hostname() == "" {
			next.ServeHTTP(w, r)
			return
		}
		host := net.JoinHostPort(target.Hostname(), port)
		p := &httputil.ReverseProxy{
			Rewrite: func(pr *httputil.ProxyRequest) {
				pr.Out.URL.Scheme, pr.Out.URL.Host = "http", host
				// The name the client used, for the portal's links and cookies.
				pr.Out.Host = pr.In.Host
				peer, _, _ := net.SplitHostPort(pr.In.RemoteAddr)
				xff := pr.In.Header.Get("X-Forwarded-For")
				if xff != "" {
					xff += ", "
				}
				pr.Out.Header.Set("X-Forwarded-For", xff+peer)
				for _, h := range []string{"X-Forwarded-Proto", "X-Forwarded-Host"} {
					if v := pr.In.Header.Get(h); v != "" {
						pr.Out.Header.Set(h, v)
					}
				}
				pr.Out.Header.Set(hdrForwardedBy, f.self)
			},
			FlushInterval: -1,
			ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
				writeForwardError(w, http.StatusBadGateway, "this control plane is standing by, and the leader did not answer: "+err.Error())
			},
		}
		p.ServeHTTP(w, r)
	})
}

// trustControlPlanes keeps the client port trusting the other control
// planes' X-Forwarded-For, besides router.trusted_proxies: a standby passes
// requests on, and the client is the address it was given, not the standby.
// Read again every half minute, for one added since.
func (f *leaderForward) trustControlPlanes(ctx context.Context, configured []string) {
	for {
		list := slices.Clone(configured)
		if cps, err := f.st.ControlPlanes(ctx); err == nil {
			for _, cp := range cps {
				if u, err := url.Parse(cp.URL); err == nil && u.Hostname() != "" {
					list = append(list, u.Hostname())
				}
			}
		}
		clientip.SetTrusted(list)
		select {
		case <-ctx.Done():
			return
		case <-time.After(30 * time.Second):
		}
	}
}

// leader is where the lease's holder is reached, and its name.
func (f *leaderForward) leader(ctx context.Context) (*url.URL, string, error) {
	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
	defer cancel()
	lease, err := f.st.Leadership(ctx, control.ControlRole)
	if err != nil || lease.Owner == "" || time.Now().After(lease.ExpiresAt) {
		return nil, "", errors.New("no control plane holds the lease right now")
	}
	name := control.HolderName(lease.Owner)
	if name == f.self {
		// This machine's previous process, whose lease has not run out.
		return nil, name, errors.New("the lease is still held by this machine's previous run; it runs out within seconds")
	}
	cps, err := f.st.ControlPlanes(ctx)
	if err != nil {
		return nil, name, err
	}
	for _, cp := range cps {
		if cp.Name == name {
			u, err := url.Parse(cp.URL)
			return u, name, err
		}
	}
	return nil, name, fmt.Errorf("%s leads, and is not among the control planes the store lists", name)
}

// leaderSide answers forwarded requests, on the agent port: only from another
// control plane of this cluster, only while this one leads, and only the API.
func (f *leaderForward) leaderSide(admin http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		path := strings.TrimPrefix(r.URL.Path, forwardPrefix)
		if adminStaysHere(path) {
			http.NotFound(w, r)
			return
		}
		if err := f.fromControlPlane(r); err != nil {
			writeForwardError(w, http.StatusForbidden, err.Error())
			return
		}
		if !f.lead.IsLeader() {
			// Not passed on again: two standbys must not bounce a request
			// between them while the lease moves.
			writeForwardError(w, http.StatusServiceUnavailable, "this control plane no longer leads; ask again")
			return
		}
		r2 := r.Clone(r.Context())
		r2.URL.Path, r2.URL.RawPath = path, ""
		w.Header().Set(hdrAnsweredBy, f.self)
		admin.ServeHTTP(w, r2)
	})
}

// fromControlPlane checks the client certificate names a control plane the
// store lists. A node's certificate is from the same authority, and a node
// must not reach the admin API by presenting it.
func (f *leaderForward) fromControlPlane(r *http.Request) error {
	if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 {
		return errors.New("a control plane's certificate is required")
	}
	cert := r.TLS.VerifiedChains[0][0]
	if !slices.Contains(cert.ExtKeyUsage, x509.ExtKeyUsageServerAuth) {
		return errors.New("not a control plane's certificate")
	}
	cps, err := f.st.ControlPlanes(r.Context())
	if err != nil {
		return err
	}
	for _, cp := range cps {
		if cp.Name == cert.Subject.CommonName {
			return nil
		}
	}
	return fmt.Errorf("%s is not a control plane of this cluster", cert.Subject.CommonName)
}

func writeForwardError(w http.ResponseWriter, code int, msg string) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
}