Files
ilkerandClaude Opus 5 ee6ef5566c hpcod: stop putting the control plane's headers on the cluster's pages
Every form in Open OnDemand came back 422. The control plane's security
headers were wrapped around everything it served, the proxy included, so
a cluster portal's responses carried two Content-Security-Policy headers,
two X-Frame-Options - DENY over the cluster's own SAMEORIGIN - and
Referrer-Policy: no-referrer.

The last one is what caused the 422. With no-referrer a browser sends
"Origin: null", and Rails' origin check rejects the request. Twelve of
them in the portal's log.

The headers now wrap the control plane's own handler, its login page
included, and the proxy sits outside. A cluster portal is a different
application and sets its own; overriding them from the outside is not
hardening, it is breaking an application whose security posture is not
this one's to decide.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-21 22:13:25 +02:00

296 lines
10 KiB
Go

package httpapi
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"math/big"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// A portal that requires a client certificate, standing in for a cluster's
// Apache with SSLVerifyClient require.
func portalRequiringAClient(t *testing.T, trust *x509.CertPool, h http.Handler) *httptest.Server {
t.Helper()
srv := httptest.NewUnstartedServer(h)
srv.TLS = &tls.Config{
ClientAuth: tls.RequireAndVerifyClientCert,
ClientCAs: trust,
}
srv.StartTLS()
t.Cleanup(srv.Close)
return srv
}
// registryFor writes a registry pinning one cluster to a server's own
// certificate, the way publishing does.
func registryFor(t *testing.T, srv *httptest.Server, label string) *ClusterRegistry {
t.Helper()
dir := t.TempDir()
certPath := filepath.Join(dir, label+".crt")
pemBytes := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw})
if err := os.WriteFile(certPath, pemBytes, 0o600); err != nil {
t.Fatal(err)
}
reg := filepath.Join(dir, "clusters.yaml")
body := "clusters:\n - id: 1\n portal: " + strings.TrimPrefix(srv.URL, "https://") +
"\n cert: " + certPath + "\n"
if err := os.WriteFile(reg, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
return NewClusterRegistry(reg, "hpc.example")
}
// The whole of the single sign-on: the portal demands a client
// certificate, the proxy presents one, and the person is never asked for
// anything.
func TestTheProxySignsInWithItsCertificate(t *testing.T) {
// The proxy's own certificate, which the portal will trust.
cert, pool := selfSigned(t, "hpcod-control-plane")
var sawCN string
srv := portalRequiringAClient(t, pool, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if len(r.TLS.PeerCertificates) > 0 {
sawCN = r.TLS.PeerCertificates[0].Subject.CommonName
}
_, _ = w.Write([]byte("the dashboard"))
}))
h := clusterProxy(registryFor(t, srv, "c001"), &cert, nil, http.NotFoundHandler())
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/pun/sys/dashboard", nil)
r.Host = "c001.hpc.example"
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, want 200; body %s", w.Code, w.Body)
}
if sawCN != "hpcod-control-plane" {
t.Errorf("the portal saw client %q, want the control plane", sawCN)
}
}
// Without the certificate the portal must refuse. That is what makes the
// arrangement safe: the proxy is not merely trusted by convention, it is
// the only client that can complete the handshake.
func TestWithoutTheCertificateThePortalRefuses(t *testing.T) {
_, pool := selfSigned(t, "hpcod-control-plane")
srv := portalRequiringAClient(t, pool, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("the dashboard"))
}))
h := clusterProxy(registryFor(t, srv, "c001"), nil, nil, http.NotFoundHandler())
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/", nil)
r.Host = "c001.hpc.example"
h.ServeHTTP(w, r)
if w.Code != http.StatusBadGateway {
t.Errorf("status = %d, want 502 - the portal should not have been reachable", w.Code)
}
}
// A different certificate is not the control plane. Anything else on the
// management network holds one of these at best.
func TestAnotherCertificateIsNotAccepted(t *testing.T) {
_, trusted := selfSigned(t, "hpcod-control-plane")
other, _ := selfSigned(t, "somebody-else")
srv := portalRequiringAClient(t, trusted, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("the dashboard"))
}))
h := clusterProxy(registryFor(t, srv, "c001"), &other, nil, http.NotFoundHandler())
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/", nil)
r.Host = "c001.hpc.example"
h.ServeHTTP(w, r)
if w.Code != http.StatusBadGateway {
t.Errorf("status = %d, want 502 - an unrelated certificate was accepted", w.Code)
}
}
// The identity comes from the certificate, so nothing a caller sends may
// look like one on the way through.
func TestUserHeadersFromTheCallerAreStripped(t *testing.T) {
cert, pool := selfSigned(t, "hpcod-control-plane")
var got http.Header
srv := portalRequiringAClient(t, pool, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got = r.Header.Clone()
}))
h := clusterProxy(registryFor(t, srv, "c001"), &cert, nil, http.NotFoundHandler())
r := httptest.NewRequest(http.MethodGet, "/", nil)
r.Host = "c001.hpc.example"
r.Header.Set("X-Forwarded-User", "root")
r.Header.Set("X-Remote-User", "root")
r.Header.Set("Remote-User", "root")
h.ServeHTTP(httptest.NewRecorder(), r)
for _, name := range []string{"X-Forwarded-User", "X-Remote-User", "Remote-User"} {
if v := got.Get(name); v != "" {
t.Errorf("%s reached the portal as %q", name, v)
}
}
}
// selfSigned makes a keypair usable both as a client certificate and as
// the thing a portal pins to recognise it - which is exactly how the real
// one is used: the control plane signs nothing, it is simply the one
// certificate each cluster has been told to accept.
func selfSigned(t *testing.T, cn string) (tls.Certificate, *x509.CertPool) {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{CommonName: cn},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(24 * time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
IsCA: true,
BasicConstraintsValid: true,
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
leaf, err := x509.ParseCertificate(der)
if err != nil {
t.Fatal(err)
}
pool := x509.NewCertPool()
pool.AddCert(leaf)
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: leaf}, pool
}
// The hole this closes, caught by fetching a cluster's dashboard from the
// internet with no session and getting it. Removing the cluster's own
// login without putting the control plane's in front of it did not make
// the portal private - it made it public.
func TestAStrangerNeverReachesAClusterPortal(t *testing.T) {
cert, pool := selfSigned(t, "hpcod-control-plane")
reached := false
srv := portalRequiringAClient(t, pool, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
reached = true
_, _ = w.Write([]byte("the dashboard"))
}))
refuse := func(*http.Request, string) bool { return false }
h := clusterProxy(registryFor(t, srv, "c001"), &cert, refuse, http.NotFoundHandler())
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/pun/sys/dashboard", nil)
r.Host = "c001.hpc.example"
h.ServeHTTP(w, r)
if reached {
t.Fatal("the request reached the cluster")
}
if w.Code != http.StatusSeeOther {
t.Errorf("status = %d, want a redirect to sign in", w.Code)
}
if loc := w.Header().Get("Location"); !strings.Contains(loc, "/login") {
t.Errorf("sent to %q, want the login", loc)
}
}
// An API client gets a status it can act on rather than a page.
func TestAnUnauthenticatedAPICallGets401(t *testing.T) {
cert, pool := selfSigned(t, "hpcod-control-plane")
srv := portalRequiringAClient(t, pool, http.NotFoundHandler())
h := clusterProxy(registryFor(t, srv, "c001"), &cert,
func(*http.Request, string) bool { return false }, http.NotFoundHandler())
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/v1/anything", nil)
r.Host = "c001.hpc.example"
h.ServeHTTP(w, r)
if w.Code != http.StatusUnauthorized {
t.Errorf("status = %d, want 401", w.Code)
}
}
// Refusing happens before the registry is consulted, so a stranger cannot
// tell a cluster that exists from one that does not.
func TestRefusalLooksTheSameForAClusterThatDoesNotExist(t *testing.T) {
dir := t.TempDir()
reg := NewClusterRegistry(filepath.Join(dir, "clusters.yaml"), "hpc.example")
h := clusterProxy(reg, nil, func(*http.Request, string) bool { return false }, http.NotFoundHandler())
codes := map[string]int{}
for _, host := range []string{"c001.hpc.example", "c099.hpc.example"} {
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/", nil)
r.Host = host
h.ServeHTTP(w, r)
codes[host] = w.Code
}
if codes["c001.hpc.example"] != codes["c099.hpc.example"] {
t.Errorf("a signed-out caller can tell the two apart: %v", codes)
}
}
// The control plane's security headers must not be added to a proxied
// cluster portal. That portal is a different application with its own:
// the duplicates gave two Content-Security-Policy and two
// X-Frame-Options headers, and Referrer-Policy: no-referrer made the
// browser send "Origin: null", so every form in Open OnDemand came back
// 422.
func TestProxiedResponsesDoNotGetTheControlPlanesHeaders(t *testing.T) {
cert, pool := selfSigned(t, "hpcod-control-plane")
srv := portalRequiringAClient(t, pool, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Frame-Options", "SAMEORIGIN") // the cluster's own
_, _ = w.Write([]byte("the dashboard"))
}))
// Assembled the way Run does: headers inside, proxy outside.
inner := securityHeaders(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("control plane"))
}))
h := clusterProxy(registryFor(t, srv, "c001"), &cert,
func(*http.Request, string) bool { return true }, inner)
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/", nil)
r.Host = "c001.hpc.example"
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("status %d", w.Code)
}
if got := w.Header().Values("X-Frame-Options"); len(got) != 1 || got[0] != "SAMEORIGIN" {
t.Errorf("X-Frame-Options = %v; the cluster's own must be the only one", got)
}
for _, name := range []string{"Referrer-Policy", "Content-Security-Policy"} {
if v := w.Header().Values(name); len(v) != 0 {
t.Errorf("%s reached a proxied response: %v", name, v)
}
}
// And the control plane's own pages still carry them.
w = httptest.NewRecorder()
r = httptest.NewRequest(http.MethodGet, "/", nil)
r.Host = "hpc.example"
h.ServeHTTP(w, r)
if w.Header().Get("Referrer-Policy") == "" || w.Header().Get("Content-Security-Policy") == "" {
t.Error("the control plane lost its own security headers")
}
}