Files
ilkerandClaude Opus 5 ecbc951c59 hpcod: a cluster portal now needs a control plane session, which it did not
Taking the cluster's own login away and letting the proxy sign in with a
certificate did not make the portal private. It made it public: the proxy
sat outside the control plane's authentication on purpose, back when the
cluster had a password of its own to fall back on. I found it by fetching
c001's dashboard from the internet with no session and getting it.

The proxy now asks before it opens a connection: a valid session, and the
cluster has to belong to whoever is asking, or be reached by an
administrator. A browser is sent to sign in, an API client gets 401, and
nothing is proxied either way.

The check runs before the registry is consulted, so a signed-out caller
gets the same answer for a cluster that exists and one that does not.

Also tagged the roles in site.yml. Changing the portal meant re-running
everything, which rebuilds the node image - four minutes and a risk taken
for nothing. Dynamic includes need "apply" for the tag to reach the
role's own tasks; a tag on the include alone only decides whether the
include runs.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-21 13:27:04 +02:00

320 lines
11 KiB
Go

package httpapi
import (
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"fmt"
"log"
"net/http"
"net/http/httputil"
"net/url"
"os"
"regexp"
"strings"
"sync"
"time"
"gopkg.in/yaml.v3"
)
// Tenant portals are reached by subdomain, not by path.
//
// Open OnDemand builds absolute URLs from the root - /pun/, /public/, /node/ -
// and has no documented base-path setting, so serving it under
// hpc.example.org/t002/ would return the login page and then send every link
// back to the control plane. It has to own the root of a hostname, so each
// tenant gets one: t002.hpc.example.org.
//
// The routing lives here rather than in Caddy because this is the only
// component that knows which tenants exist. Caddy keeps one static wildcard
// block and never has to be rewritten when a tenant is created or destroyed -
// which also means provisioning never needs write access to the web server's
// configuration.
// A conservative label: a cluster id is generated, never user-supplied, and
// anything else is a request for a host we do not serve. Matching narrowly
// keeps a crafted Host header from steering the proxy somewhere unintended.
//
// Clusters, not tenants. One tenant may hold several at once, so the name
// that has to be unique is the cluster's.
var clusterLabel = regexp.MustCompile(`^c[0-9]{3}$`)
// ClusterRegistry maps a tenant's subdomain label to the head node behind it.
//
// Backed by a file the provisioner writes, re-read when it changes, because a
// tenant created after this process started must be reachable without
// restarting the control plane.
type ClusterRegistry struct {
path string
domain string
mu sync.RWMutex
loaded time.Time
modTime time.Time
clusters map[string]clusterTarget // label -> where its portal is
}
type clusterFile struct {
Clusters []struct {
ID int `yaml:"id"`
Portal string `yaml:"portal"` // host or host:port of the tenant's portal
// PEM of the certificate that portal presents. A tenant signs its
// own, so this file is the trust anchor for that one tenant and
// nothing else - which is stricter than a shared CA would be, since
// a tenant's certificate cannot vouch for any other tenant.
Cert string `yaml:"cert"`
} `yaml:"clusters"`
}
// clusterTarget is where a tenant's portal lives and what certificate it is
// expected to present.
type clusterTarget struct {
addr string
pool *x509.CertPool
// Fingerprint of the certificate the pool was built from. It is part of
// the transport cache key: without it a transport keyed on the address
// alone outlives the certificate it trusts, and a tenant that rotates
// its certificate stays unreachable until the control plane restarts.
// That happened - the first version cached on address only.
certSum string
}
// NewClusterRegistry returns a registry over path. A missing file is not an
// error: it means no tenant has been published yet, and the control plane
// still has to serve its own pages.
func NewClusterRegistry(path, domain string) *ClusterRegistry {
return &ClusterRegistry{path: path, domain: strings.ToLower(domain)}
}
func (r *ClusterRegistry) reload() {
st, err := os.Stat(r.path)
if err != nil {
r.mu.Lock()
r.clusters = nil
r.mu.Unlock()
return
}
r.mu.RLock()
fresh := st.ModTime().Equal(r.modTime) && r.clusters != nil
r.mu.RUnlock()
if fresh {
return
}
raw, err := os.ReadFile(r.path)
if err != nil {
log.Printf("tenant registry: %v", err)
return
}
var cf clusterFile
if err := yaml.Unmarshal(raw, &cf); err != nil {
// Keep serving the previous map. A malformed file should not take
// every tenant portal offline at once.
log.Printf("tenant registry: %v (keeping previous contents)", err)
return
}
m := make(map[string]clusterTarget, len(cf.Clusters))
for _, t := range cf.Clusters {
if t.Portal == "" {
continue
}
pool := x509.NewCertPool()
pem, err := os.ReadFile(t.Cert)
if err != nil || !pool.AppendCertsFromPEM(pem) {
// Without the certificate there is no way to tell the tenant's
// portal from anything else answering on that address, so the
// tenant is left unpublished rather than proxied blindly.
log.Printf("cluster registry: c%03d: no usable certificate at %q; not publishing", t.ID, t.Cert)
continue
}
sum := sha256.Sum256(pem)
m[fmt.Sprintf("c%03d", t.ID)] = clusterTarget{
addr: t.Portal,
pool: pool,
certSum: hex.EncodeToString(sum[:]),
}
}
r.mu.Lock()
r.clusters = m
r.modTime = st.ModTime()
r.loaded = time.Now()
r.mu.Unlock()
}
// lookup resolves a Host header to a tenant's portal address. The second
// return value is false for the control plane's own hostname and for anything
// that is not a tenant we know about.
func (r *ClusterRegistry) lookup(host string) (clusterTarget, bool) {
host = strings.ToLower(host)
if i := strings.IndexByte(host, ':'); i >= 0 {
host = host[:i]
}
suffix := "." + r.domain
if !strings.HasSuffix(host, suffix) {
return clusterTarget{}, false
}
label := strings.TrimSuffix(host, suffix)
if !clusterLabel.MatchString(label) {
return clusterTarget{}, false
}
r.reload()
r.mu.RLock()
defer r.mu.RUnlock()
t, ok := r.clusters[label]
return t, ok
}
// names reports whether a host is a cluster subdomain at all, regardless
// of whether that cluster exists.
//
// The difference matters. Falling through sends an unknown cluster to the
// control plane, which answers with its own front page - so a destroyed
// or not-yet-published cluster looks like a working site at the wrong
// address, which is worse than an error.
func (r *ClusterRegistry) names(host string) (string, bool) {
host = strings.ToLower(host)
if i := strings.IndexByte(host, ':'); i >= 0 {
host = host[:i]
}
suffix := "." + r.domain
if !strings.HasSuffix(host, suffix) {
return "", false
}
label := strings.TrimSuffix(host, suffix)
if !clusterLabel.MatchString(label) {
return "", false
}
return label, true
}
// clusterProxy routes by Host: a known tenant subdomain is proxied to that
// tenant's portal, everything else falls through to the control plane's own
// handler.
// clientCert is what the control plane presents to a cluster's portal.
//
// It is the whole of the single sign-on: the cluster requires a client
// certificate and trusts exactly this one, so a request that arrives
// through this proxy is already authenticated and the portal asks for
// nothing. Anything else on the management network cannot complete the
// handshake at all.
//
// Optional. Without it the proxy still works against a portal that does
// not ask, which is what every cluster built before this did.
// gate decides whether a request may reach a cluster's portal at all.
//
// It has to exist. The portal used to hold its own login, so the proxy
// could pass anyone through and let the cluster decide. Now that the
// cluster accepts the proxy's certificate instead, passing anyone through
// means passing everyone in - the first version of this change did
// exactly that, and the cluster's dashboard answered an unauthenticated
// request from the internet.
//
// Returning false means the caller is sent to sign in; the proxy never
// opens a connection to the cluster for them.
type gate func(r *http.Request, label string) bool
func clusterProxy(reg *ClusterRegistry, clientCert *tls.Certificate, allow gate, next http.Handler) http.Handler {
// One transport per tenant, each trusting exactly that tenant's own
// certificate and nothing else.
//
// A tenant signs its own certificate for its management IP, so the public
// CAs cannot vouch for it - but that does not mean the hop should go
// unverified. Pinning the tenant's certificate as the sole root for that
// tenant is stricter than a shared internal CA would be: a certificate
// belonging to one tenant cannot authenticate another, and a machine that
// takes over the address cannot answer for it at all.
var mu sync.Mutex
transports := map[string]*http.Transport{}
transportFor := func(t clusterTarget) *http.Transport {
mu.Lock()
defer mu.Unlock()
key := t.addr + "|" + t.certSum
if tr, ok := transports[key]; ok {
return tr
}
cfg := &tls.Config{RootCAs: t.pool, MinVersion: tls.VersionTLS12}
if clientCert != nil {
cfg.Certificates = []tls.Certificate{*clientCert}
}
tr := &http.Transport{
TLSClientConfig: cfg,
TLSHandshakeTimeout: 10 * time.Second,
IdleConnTimeout: 90 * time.Second,
}
// A rotation leaves the previous transport behind. Bounded by the
// number of tenants times how often they rotate, and each one is a
// few idle connections that time out on their own, so it is not
// worth the bookkeeping to prune.
transports[key] = tr
return tr
}
newProxy := func(t clusterTarget) *httputil.ReverseProxy {
return &httputil.ReverseProxy{
Transport: transportFor(t),
// Rewrite, not Director: ReverseProxy requires exactly one, and
// Rewrite is the one that exposes the inbound request, which is
// where the original Host comes from.
Rewrite: func(pr *httputil.ProxyRequest) {
pr.SetURL(&url.URL{Scheme: "https", Host: t.addr})
// OnDemand generates absolute URLs from the Host it is given.
// Passing the original through keeps every link it emits
// pointing at the public name rather than at an address
// nobody outside can reach.
pr.Out.Host = pr.In.Host
pr.SetXForwarded()
// The portal takes its user from the verified client
// certificate, so nothing a caller sends can name one.
// Stripping these anyway costs nothing and closes the
// door if a cluster is ever configured to read them.
pr.Out.Header.Del("X-Forwarded-User")
pr.Out.Header.Del("X-Remote-User")
pr.Out.Header.Del("Remote-User")
},
ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
// A tenant that is provisioning, rebooting or already
// destroyed is an expected state, not a control-plane fault.
log.Printf("tenant proxy: %s: %v", r.Host, err)
http.Error(w, "tenant portal is not reachable", http.StatusBadGateway)
},
}
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
label, isCluster := reg.names(r.Host)
if isCluster && allow != nil && !allow(r, label) {
// Checked before the cluster is looked up, so a stranger
// cannot tell a cluster that exists from one that does not.
if strings.HasPrefix(r.URL.Path, "/v1/") ||
strings.Contains(r.Header.Get("Accept"), "application/json") {
http.Error(w, "sign in to the control plane first", http.StatusUnauthorized)
return
}
http.Redirect(w, r, "https://"+reg.domain+"/login", http.StatusSeeOther)
return
}
target, ok := reg.lookup(r.Host)
if !ok {
if isCluster {
// Named like a cluster, and there is no such cluster.
// Saying so beats handing back the control plane's front
// page, which reads as "this works" at an address that
// does not.
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
http.Error(w, "no cluster "+label+" is published here", http.StatusNotFound)
return
}
next.ServeHTTP(w, r)
return
}
newProxy(target).ServeHTTP(w, r)
})
}