Taking the cluster's own login away and letting the proxy sign in with a certificate did not make the portal private. It made it public: the proxy sat outside the control plane's authentication on purpose, back when the cluster had a password of its own to fall back on. I found it by fetching c001's dashboard from the internet with no session and getting it. The proxy now asks before it opens a connection: a valid session, and the cluster has to belong to whoever is asking, or be reached by an administrator. A browser is sent to sign in, an API client gets 401, and nothing is proxied either way. The check runs before the registry is consulted, so a signed-out caller gets the same answer for a cluster that exists and one that does not. Also tagged the roles in site.yml. Changing the portal meant re-running everything, which rebuilds the node image - four minutes and a risk taken for nothing. Dynamic includes need "apply" for the tag to reach the role's own tasks; a tag on the include alone only decides whether the include runs. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
320 lines
11 KiB
Go
320 lines
11 KiB
Go
package httpapi
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"net/http/httputil"
|
|
"net/url"
|
|
"os"
|
|
"regexp"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
// Tenant portals are reached by subdomain, not by path.
|
|
//
|
|
// Open OnDemand builds absolute URLs from the root - /pun/, /public/, /node/ -
|
|
// and has no documented base-path setting, so serving it under
|
|
// hpc.example.org/t002/ would return the login page and then send every link
|
|
// back to the control plane. It has to own the root of a hostname, so each
|
|
// tenant gets one: t002.hpc.example.org.
|
|
//
|
|
// The routing lives here rather than in Caddy because this is the only
|
|
// component that knows which tenants exist. Caddy keeps one static wildcard
|
|
// block and never has to be rewritten when a tenant is created or destroyed -
|
|
// which also means provisioning never needs write access to the web server's
|
|
// configuration.
|
|
|
|
// A conservative label: a cluster id is generated, never user-supplied, and
|
|
// anything else is a request for a host we do not serve. Matching narrowly
|
|
// keeps a crafted Host header from steering the proxy somewhere unintended.
|
|
//
|
|
// Clusters, not tenants. One tenant may hold several at once, so the name
|
|
// that has to be unique is the cluster's.
|
|
var clusterLabel = regexp.MustCompile(`^c[0-9]{3}$`)
|
|
|
|
// ClusterRegistry maps a tenant's subdomain label to the head node behind it.
|
|
//
|
|
// Backed by a file the provisioner writes, re-read when it changes, because a
|
|
// tenant created after this process started must be reachable without
|
|
// restarting the control plane.
|
|
type ClusterRegistry struct {
|
|
path string
|
|
domain string
|
|
|
|
mu sync.RWMutex
|
|
loaded time.Time
|
|
modTime time.Time
|
|
clusters map[string]clusterTarget // label -> where its portal is
|
|
}
|
|
|
|
type clusterFile struct {
|
|
Clusters []struct {
|
|
ID int `yaml:"id"`
|
|
Portal string `yaml:"portal"` // host or host:port of the tenant's portal
|
|
// PEM of the certificate that portal presents. A tenant signs its
|
|
// own, so this file is the trust anchor for that one tenant and
|
|
// nothing else - which is stricter than a shared CA would be, since
|
|
// a tenant's certificate cannot vouch for any other tenant.
|
|
Cert string `yaml:"cert"`
|
|
} `yaml:"clusters"`
|
|
}
|
|
|
|
// clusterTarget is where a tenant's portal lives and what certificate it is
|
|
// expected to present.
|
|
type clusterTarget struct {
|
|
addr string
|
|
pool *x509.CertPool
|
|
// Fingerprint of the certificate the pool was built from. It is part of
|
|
// the transport cache key: without it a transport keyed on the address
|
|
// alone outlives the certificate it trusts, and a tenant that rotates
|
|
// its certificate stays unreachable until the control plane restarts.
|
|
// That happened - the first version cached on address only.
|
|
certSum string
|
|
}
|
|
|
|
// NewClusterRegistry returns a registry over path. A missing file is not an
|
|
// error: it means no tenant has been published yet, and the control plane
|
|
// still has to serve its own pages.
|
|
func NewClusterRegistry(path, domain string) *ClusterRegistry {
|
|
return &ClusterRegistry{path: path, domain: strings.ToLower(domain)}
|
|
}
|
|
|
|
func (r *ClusterRegistry) reload() {
|
|
st, err := os.Stat(r.path)
|
|
if err != nil {
|
|
r.mu.Lock()
|
|
r.clusters = nil
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
|
|
r.mu.RLock()
|
|
fresh := st.ModTime().Equal(r.modTime) && r.clusters != nil
|
|
r.mu.RUnlock()
|
|
if fresh {
|
|
return
|
|
}
|
|
|
|
raw, err := os.ReadFile(r.path)
|
|
if err != nil {
|
|
log.Printf("tenant registry: %v", err)
|
|
return
|
|
}
|
|
var cf clusterFile
|
|
if err := yaml.Unmarshal(raw, &cf); err != nil {
|
|
// Keep serving the previous map. A malformed file should not take
|
|
// every tenant portal offline at once.
|
|
log.Printf("tenant registry: %v (keeping previous contents)", err)
|
|
return
|
|
}
|
|
|
|
m := make(map[string]clusterTarget, len(cf.Clusters))
|
|
for _, t := range cf.Clusters {
|
|
if t.Portal == "" {
|
|
continue
|
|
}
|
|
pool := x509.NewCertPool()
|
|
pem, err := os.ReadFile(t.Cert)
|
|
if err != nil || !pool.AppendCertsFromPEM(pem) {
|
|
// Without the certificate there is no way to tell the tenant's
|
|
// portal from anything else answering on that address, so the
|
|
// tenant is left unpublished rather than proxied blindly.
|
|
log.Printf("cluster registry: c%03d: no usable certificate at %q; not publishing", t.ID, t.Cert)
|
|
continue
|
|
}
|
|
sum := sha256.Sum256(pem)
|
|
m[fmt.Sprintf("c%03d", t.ID)] = clusterTarget{
|
|
addr: t.Portal,
|
|
pool: pool,
|
|
certSum: hex.EncodeToString(sum[:]),
|
|
}
|
|
}
|
|
|
|
r.mu.Lock()
|
|
r.clusters = m
|
|
r.modTime = st.ModTime()
|
|
r.loaded = time.Now()
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
// lookup resolves a Host header to a tenant's portal address. The second
|
|
// return value is false for the control plane's own hostname and for anything
|
|
// that is not a tenant we know about.
|
|
func (r *ClusterRegistry) lookup(host string) (clusterTarget, bool) {
|
|
host = strings.ToLower(host)
|
|
if i := strings.IndexByte(host, ':'); i >= 0 {
|
|
host = host[:i]
|
|
}
|
|
suffix := "." + r.domain
|
|
if !strings.HasSuffix(host, suffix) {
|
|
return clusterTarget{}, false
|
|
}
|
|
label := strings.TrimSuffix(host, suffix)
|
|
if !clusterLabel.MatchString(label) {
|
|
return clusterTarget{}, false
|
|
}
|
|
|
|
r.reload()
|
|
r.mu.RLock()
|
|
defer r.mu.RUnlock()
|
|
t, ok := r.clusters[label]
|
|
return t, ok
|
|
}
|
|
|
|
// names reports whether a host is a cluster subdomain at all, regardless
|
|
// of whether that cluster exists.
|
|
//
|
|
// The difference matters. Falling through sends an unknown cluster to the
|
|
// control plane, which answers with its own front page - so a destroyed
|
|
// or not-yet-published cluster looks like a working site at the wrong
|
|
// address, which is worse than an error.
|
|
func (r *ClusterRegistry) names(host string) (string, bool) {
|
|
host = strings.ToLower(host)
|
|
if i := strings.IndexByte(host, ':'); i >= 0 {
|
|
host = host[:i]
|
|
}
|
|
suffix := "." + r.domain
|
|
if !strings.HasSuffix(host, suffix) {
|
|
return "", false
|
|
}
|
|
label := strings.TrimSuffix(host, suffix)
|
|
if !clusterLabel.MatchString(label) {
|
|
return "", false
|
|
}
|
|
return label, true
|
|
}
|
|
|
|
// clusterProxy routes by Host: a known tenant subdomain is proxied to that
|
|
// tenant's portal, everything else falls through to the control plane's own
|
|
// handler.
|
|
// clientCert is what the control plane presents to a cluster's portal.
|
|
//
|
|
// It is the whole of the single sign-on: the cluster requires a client
|
|
// certificate and trusts exactly this one, so a request that arrives
|
|
// through this proxy is already authenticated and the portal asks for
|
|
// nothing. Anything else on the management network cannot complete the
|
|
// handshake at all.
|
|
//
|
|
// Optional. Without it the proxy still works against a portal that does
|
|
// not ask, which is what every cluster built before this did.
|
|
// gate decides whether a request may reach a cluster's portal at all.
|
|
//
|
|
// It has to exist. The portal used to hold its own login, so the proxy
|
|
// could pass anyone through and let the cluster decide. Now that the
|
|
// cluster accepts the proxy's certificate instead, passing anyone through
|
|
// means passing everyone in - the first version of this change did
|
|
// exactly that, and the cluster's dashboard answered an unauthenticated
|
|
// request from the internet.
|
|
//
|
|
// Returning false means the caller is sent to sign in; the proxy never
|
|
// opens a connection to the cluster for them.
|
|
type gate func(r *http.Request, label string) bool
|
|
|
|
func clusterProxy(reg *ClusterRegistry, clientCert *tls.Certificate, allow gate, next http.Handler) http.Handler {
|
|
// One transport per tenant, each trusting exactly that tenant's own
|
|
// certificate and nothing else.
|
|
//
|
|
// A tenant signs its own certificate for its management IP, so the public
|
|
// CAs cannot vouch for it - but that does not mean the hop should go
|
|
// unverified. Pinning the tenant's certificate as the sole root for that
|
|
// tenant is stricter than a shared internal CA would be: a certificate
|
|
// belonging to one tenant cannot authenticate another, and a machine that
|
|
// takes over the address cannot answer for it at all.
|
|
var mu sync.Mutex
|
|
transports := map[string]*http.Transport{}
|
|
transportFor := func(t clusterTarget) *http.Transport {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
key := t.addr + "|" + t.certSum
|
|
if tr, ok := transports[key]; ok {
|
|
return tr
|
|
}
|
|
cfg := &tls.Config{RootCAs: t.pool, MinVersion: tls.VersionTLS12}
|
|
if clientCert != nil {
|
|
cfg.Certificates = []tls.Certificate{*clientCert}
|
|
}
|
|
tr := &http.Transport{
|
|
TLSClientConfig: cfg,
|
|
TLSHandshakeTimeout: 10 * time.Second,
|
|
IdleConnTimeout: 90 * time.Second,
|
|
}
|
|
// A rotation leaves the previous transport behind. Bounded by the
|
|
// number of tenants times how often they rotate, and each one is a
|
|
// few idle connections that time out on their own, so it is not
|
|
// worth the bookkeeping to prune.
|
|
transports[key] = tr
|
|
return tr
|
|
}
|
|
|
|
newProxy := func(t clusterTarget) *httputil.ReverseProxy {
|
|
return &httputil.ReverseProxy{
|
|
Transport: transportFor(t),
|
|
// Rewrite, not Director: ReverseProxy requires exactly one, and
|
|
// Rewrite is the one that exposes the inbound request, which is
|
|
// where the original Host comes from.
|
|
Rewrite: func(pr *httputil.ProxyRequest) {
|
|
pr.SetURL(&url.URL{Scheme: "https", Host: t.addr})
|
|
// OnDemand generates absolute URLs from the Host it is given.
|
|
// Passing the original through keeps every link it emits
|
|
// pointing at the public name rather than at an address
|
|
// nobody outside can reach.
|
|
pr.Out.Host = pr.In.Host
|
|
pr.SetXForwarded()
|
|
// The portal takes its user from the verified client
|
|
// certificate, so nothing a caller sends can name one.
|
|
// Stripping these anyway costs nothing and closes the
|
|
// door if a cluster is ever configured to read them.
|
|
pr.Out.Header.Del("X-Forwarded-User")
|
|
pr.Out.Header.Del("X-Remote-User")
|
|
pr.Out.Header.Del("Remote-User")
|
|
},
|
|
ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
|
|
// A tenant that is provisioning, rebooting or already
|
|
// destroyed is an expected state, not a control-plane fault.
|
|
log.Printf("tenant proxy: %s: %v", r.Host, err)
|
|
http.Error(w, "tenant portal is not reachable", http.StatusBadGateway)
|
|
},
|
|
}
|
|
}
|
|
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
label, isCluster := reg.names(r.Host)
|
|
if isCluster && allow != nil && !allow(r, label) {
|
|
// Checked before the cluster is looked up, so a stranger
|
|
// cannot tell a cluster that exists from one that does not.
|
|
if strings.HasPrefix(r.URL.Path, "/v1/") ||
|
|
strings.Contains(r.Header.Get("Accept"), "application/json") {
|
|
http.Error(w, "sign in to the control plane first", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
http.Redirect(w, r, "https://"+reg.domain+"/login", http.StatusSeeOther)
|
|
return
|
|
}
|
|
|
|
target, ok := reg.lookup(r.Host)
|
|
if !ok {
|
|
if isCluster {
|
|
// Named like a cluster, and there is no such cluster.
|
|
// Saying so beats handing back the control plane's front
|
|
// page, which reads as "this works" at an address that
|
|
// does not.
|
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
http.Error(w, "no cluster "+label+" is published here", http.StatusNotFound)
|
|
return
|
|
}
|
|
next.ServeHTTP(w, r)
|
|
return
|
|
}
|
|
newProxy(target).ServeHTTP(w, r)
|
|
})
|
|
}
|