Add a private CA and mutual TLS for the agent stream

A node's identity was whatever the agent said it was. With mutual TLS the
control plane compares the client certificate's common name against the
claimed node id and refuses the connection when they disagree, so identity
becomes something a node proves rather than something it picks.

A public authority cannot supply these certificates: Let's Encrypt and its
peers stopped issuing the clientAuth usage, and Go verifies client chains
against it. Hence internal/tlsconf/ca.go, a small authority whose key lives
on one machine and whose root is trusted everywhere, and `llmgw ctl cert`
over it so issuing is reproducible instead of remembered openssl.

The server certificate's hosts matter more than they look: hostname
verification reads subject alternative names and ignores the common name
entirely, so a certificate without the address agents dial — bare IP
included — is rejected no matter what its subject says. There is a test for
exactly that, because it is the failure everyone hits once.

requireIdentity stays wired to false until the configuration lands, so this
commit changes no running behaviour.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

internal/agent/agent_test.go

@@ -208,7 +208,7 @@ func startWith(t *testing.T, log *slog.Logger, cfgFns ...func(*Config)) *harness
 
 	inv := inventory.NewStore()
 	rec := newRecorder()
-	svc := agentsvc.New(inv, rec, log)
+	svc := agentsvc.New(inv, rec, false, log)
 
 	path, handler := svc.Handler()
 	mux := http.NewServeMux()

internal/cli/cert.go

@@ -0,0 +1,254 @@
+package cli
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
+)
+
+// defaultPKIDir is where the authority lives on the machine that issues
+// certificates. Only that machine ever holds ca-key.pem; everything else gets
+// ca.pem and its own leaf.
+const defaultPKIDir = "/etc/llmgw/pki"
+
+func newCertCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "cert",
+		Short: "Issue the certificates the control plane and agents authenticate with",
+		Long: "llmgw's internal connections use mutual TLS, and a public authority cannot\n" +
+			"supply them: Let's Encrypt and its peers stopped issuing certificates\n" +
+			"carrying the clientAuth usage, so nothing they hand out can authenticate an\n" +
+			"agent. These commands run a small private authority instead.\n\n" +
+			"They work on files and talk to nothing, so they run wherever you keep the\n" +
+			"authority — not necessarily on a cluster machine.",
+	}
+	cmd.AddCommand(newInitCACmd(), newIssueServerCmd(), newIssueClientCmd(), newCertInspectCmd())
+	return cmd
+}
+
+func newInitCACmd() *cobra.Command {
+	var (
+		dir   string
+		cn    string
+		force bool
+	)
+	cmd := &cobra.Command{
+		Use:   "init-ca",
+		Short: "Create the cluster's certificate authority",
+		Long: "Writes ca.pem and ca-key.pem. Copy ca.pem to every machine; leave\n" +
+			"ca-key.pem here, because whoever holds it can mint any node's identity.",
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			keyPath := filepath.Join(dir, "ca-key.pem")
+			if _, err := os.Stat(keyPath); err == nil && !force {
+				// Overwriting the key orphans every certificate ever issued from
+				// it, across every machine, with no warning until connections
+				// start failing.
+				return fmt.Errorf("%s already exists; re-running would invalidate every "+
+					"certificate issued from it (pass --force if that is what you want)", keyPath)
+			}
+
+			ca, err := tlsconf.NewCA(cn)
+			if err != nil {
+				return err
+			}
+			if err := ca.Write(dir); err != nil {
+				return err
+			}
+			cmd.Printf("authority %q written to %s\n", cn, dir)
+			cmd.Printf("  ca.pem      trust this everywhere (valid until %s)\n",
+				ca.Cert.NotAfter.Format(time.DateOnly))
+			cmd.Printf("  ca-key.pem  keep it here, mode 0600\n")
+			return nil
+		},
+	}
+	cmd.Flags().StringVar(&dir, "dir", defaultPKIDir, "directory holding the authority")
+	cmd.Flags().StringVar(&cn, "cn", "llmgw cluster CA", "common name for the authority")
+	cmd.Flags().BoolVar(&force, "force", false, "replace an existing authority")
+	return cmd
+}
+
+func newIssueServerCmd() *cobra.Command {
+	var (
+		dir   string
+		out   string
+		hosts []string
+		name  string
+	)
+	cmd := &cobra.Command{
+		Use:   "issue-server <common-name>",
+		Short: "Issue the control plane's certificate",
+		Long: "Every address an agent might dial must appear in --host, including bare\n" +
+			"IP addresses. Hostname verification reads subject alternative names and\n" +
+			"ignores the common name entirely, so a certificate that names the host\n" +
+			"only in its subject is rejected by every client.",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ca, err := loadCA(dir)
+			if err != nil {
+				return err
+			}
+			leaf, err := ca.IssueServer(args[0], hosts)
+			if err != nil {
+				return err
+			}
+			if name == "" {
+				name = "server"
+			}
+			if err := leaf.Write(out, name); err != nil {
+				return err
+			}
+			return describeIssued(cmd, out, name, hosts)
+		},
+	}
+	cmd.Flags().StringVar(&dir, "dir", defaultPKIDir, "directory holding the authority")
+	cmd.Flags().StringVar(&out, "out", ".", "directory to write the certificate into")
+	cmd.Flags().StringVar(&name, "name", "server", "base filename for the certificate")
+	cmd.Flags().StringArrayVar(&hosts, "host", nil, "address agents dial; repeatable (IP or DNS name)")
+	return cmd
+}
+
+func newIssueClientCmd() *cobra.Command {
+	var dir, out string
+	cmd := &cobra.Command{
+		Use:   "issue-client <node-id>",
+		Short: "Issue one agent's certificate",
+		Long: "The node id becomes the certificate's common name, and the control plane\n" +
+			"refuses any agent whose claimed id does not match it. That is what makes a\n" +
+			"node's identity something it proves rather than something it picks.",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ca, err := loadCA(dir)
+			if err != nil {
+				return err
+			}
+			leaf, err := ca.IssueClient(args[0])
+			if err != nil {
+				return err
+			}
+			if err := leaf.Write(out, args[0]); err != nil {
+				return err
+			}
+			return describeIssued(cmd, out, args[0], nil)
+		},
+	}
+	cmd.Flags().StringVar(&dir, "dir", defaultPKIDir, "directory holding the authority")
+	cmd.Flags().StringVar(&out, "out", ".", "directory to write the certificate into")
+	return cmd
+}
+
+func newCertInspectCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "inspect <file.pem>",
+		Short: "Print what a certificate actually says",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cert, err := readCert(args[0])
+			if err != nil {
+				return err
+			}
+
+			cmd.Printf("subject    %s\n", cert.Subject)
+			cmd.Printf("issuer     %s\n", cert.Issuer)
+			cmd.Printf("valid      %s .. %s", cert.NotBefore.Format(time.DateOnly),
+				cert.NotAfter.Format(time.DateOnly))
+			if d := time.Until(cert.NotAfter); d < 0 {
+				cmd.Printf("  (EXPIRED)")
+			} else {
+				cmd.Printf("  (%d days left)", int(d.Hours()/24))
+			}
+			cmd.Println()
+
+			if cert.IsCA {
+				cmd.Println("role       certificate authority")
+			}
+			if u := usageNames(cert.ExtKeyUsage); u != "" {
+				cmd.Printf("usage      %s\n", u)
+			}
+			if sans := sanNames(cert); sans != "" {
+				cmd.Printf("hosts      %s\n", sans)
+			} else if !cert.IsCA && !hasUsage(cert, x509.ExtKeyUsageClientAuth) {
+				cmd.Println("hosts      none — no client will accept this as a server certificate")
+			}
+			return nil
+		},
+	}
+}
+
+func loadCA(dir string) (*tlsconf.CA, error) {
+	ca, err := tlsconf.LoadCA(filepath.Join(dir, "ca.pem"), filepath.Join(dir, "ca-key.pem"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, fmt.Errorf("no authority in %s — run 'llmgw ctl cert init-ca --dir %s' first", dir, dir)
+		}
+		return nil, err
+	}
+	return ca, nil
+}
+
+func describeIssued(cmd *cobra.Command, dir, name string, hosts []string) error {
+	cert, err := readCert(filepath.Join(dir, name+".pem"))
+	if err != nil {
+		return err
+	}
+	cmd.Printf("issued %s\n", cert.Subject.CommonName)
+	cmd.Printf("  %s\n", filepath.Join(dir, name+".pem"))
+	cmd.Printf("  %s\n", filepath.Join(dir, name+"-key.pem"))
+	if len(hosts) > 0 {
+		cmd.Printf("  hosts: %s\n", strings.Join(hosts, ", "))
+	}
+	cmd.Printf("  expires %s\n", cert.NotAfter.Format(time.DateOnly))
+	return nil
+}
+
+func readCert(path string) (*x509.Certificate, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	block, _ := pem.Decode(b)
+	if block == nil {
+		return nil, fmt.Errorf("%s is not PEM", path)
+	}
+	return x509.ParseCertificate(block.Bytes)
+}
+
+func hasUsage(cert *x509.Certificate, want x509.ExtKeyUsage) bool {
+	for _, u := range cert.ExtKeyUsage {
+		if u == want {
+			return true
+		}
+	}
+	return false
+}
+
+func usageNames(usages []x509.ExtKeyUsage) string {
+	var out []string
+	for _, u := range usages {
+		switch u {
+		case x509.ExtKeyUsageServerAuth:
+			out = append(out, "serverAuth (identifies the control plane)")
+		case x509.ExtKeyUsageClientAuth:
+			out = append(out, "clientAuth (identifies an agent)")
+		default:
+			out = append(out, fmt.Sprintf("%v", u))
+		}
+	}
+	return strings.Join(out, ", ")
+}
+
+func sanNames(cert *x509.Certificate) string {
+	names := append([]string{}, cert.DNSNames...)
+	for _, ip := range cert.IPAddresses {
+		names = append(names, ip.String())
+	}
+	return strings.Join(names, ", ")
+}

internal/cli/ctl.go

@@ -106,6 +106,9 @@ func newCtlCmd() *cobra.Command {
 			"Allow placement on a node again"),
 		newNodeStateCmd(client, "drain", "draining",
 			"Move work off a node, one replica at a time"),
+		// cert works on files rather than the admin API, so --server does not
+		// apply to it; it lives here because this is where operators look.
+		newCertCmd(),
 	)
 	return cmd
 }

internal/cli/server.go

@@ -93,7 +93,7 @@ func newServerCmd() *cobra.Command {
 					return err
 				}
 
-				ctl, svc, err := startControlPlane(ctx, cfg, cat, rt, log)
+				ctl, svc, err := startControlPlane(ctx, cfg, cat, rt, false, log)
 				if err != nil {
 					return err
 				}
@@ -196,7 +196,7 @@ func newServerCmd() *cobra.Command {
 // it holds the open streams, so it is the only thing that can deliver a command
 // to a node.
 func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Catalog,
-	sink control.EndpointSink, log *slog.Logger,
+	sink control.EndpointSink, requireIdentity bool, log *slog.Logger,
 ) (*control.Controller, *agentsvc.Service, error) {
 	ccfg := control.DefaultConfig()
 	ccfg.Reconciler = reconciler.Config{
@@ -217,7 +217,7 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Cata
 	// the service once it exists.
 	fwd := &dispatcherRef{}
 	ctl := control.New(ccfg, cat, fwd, sink, log)
-	svc := agentsvc.New(ctl.Inventory(), ctl.Reconciler(), log)
+	svc := agentsvc.New(ctl.Inventory(), ctl.Reconciler(), requireIdentity, log)
 	fwd.set(svc)
 
 	ctl.Start(ctx)

internal/control/agentsvc/identity.go

@@ -0,0 +1,57 @@
+package agentsvc
+
+import (
+	"context"
+	"crypto/x509"
+	"net/http"
+)
+
+// peerKey carries the verified client identity through to the stream handler.
+//
+// connect's BidiStream exposes the peer's address but not its certificate, so
+// the identity is lifted out of the TLS state by middleware and put on the
+// context before the connect handler ever sees the request.
+type peerKey struct{}
+
+// PeerIdentity is who the other end proved itself to be.
+type PeerIdentity struct {
+	// CommonName is the certificate's subject CN, which is the node id a node
+	// is allowed to claim.
+	CommonName string
+	// Verified is false when the connection carried no client certificate,
+	// which only happens when TLS is switched off.
+	Verified bool
+}
+
+// WithPeerIdentity records a verified identity on the context.
+func WithPeerIdentity(ctx context.Context, id PeerIdentity) context.Context {
+	return context.WithValue(ctx, peerKey{}, id)
+}
+
+// peerFrom reads the identity a request was authenticated with.
+func peerFrom(ctx context.Context) PeerIdentity {
+	id, _ := ctx.Value(peerKey{}).(PeerIdentity)
+	return id
+}
+
+// AuthMiddleware lifts the client certificate's common name onto the request
+// context.
+//
+// The certificate chain has already been verified by the TLS handshake — this
+// only makes the result reachable from the handler. Without a certificate the
+// identity is marked unverified and the handler decides what to do, which keeps
+// the insecure development path working without a second code path through the
+// service itself.
+func AuthMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var id PeerIdentity
+		if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 {
+			id = identityOf(r.TLS.PeerCertificates[0])
+		}
+		next.ServeHTTP(w, r.WithContext(WithPeerIdentity(r.Context(), id)))
+	})
+}
+
+func identityOf(cert *x509.Certificate) PeerIdentity {
+	return PeerIdentity{CommonName: cert.Subject.CommonName, Verified: true}
+}

internal/control/agentsvc/service.go

@@ -34,6 +34,10 @@ type Service struct {
 	status StatusSink
 	log    *slog.Logger
 
+	// requireIdentity refuses agents that present no certificate. It is off
+	// only for the insecure development path.
+	requireIdentity bool
+
 	mu    sync.RWMutex
 	conns map[string]*conn
 }
@@ -52,16 +56,28 @@ type conn struct {
 func (c *conn) close() { c.once.Do(func() { close(c.closed) }) }
 
 // New builds the service.
-func New(inv *inventory.Store, status StatusSink, log *slog.Logger) *Service {
+//
+// requireIdentity refuses any agent that has not proved who it is. It should be
+// true whenever TLS is configured, and is false only for local runs without
+// certificates.
+func New(inv *inventory.Store, status StatusSink, requireIdentity bool, log *slog.Logger) *Service {
 	if log == nil {
 		log = slog.Default()
 	}
-	return &Service{inv: inv, status: status, log: log, conns: map[string]*conn{}}
+	return &Service{
+		inv: inv, status: status, log: log,
+		requireIdentity: requireIdentity,
+		conns:           map[string]*conn{},
+	}
 }
 
 // Handler returns the HTTP path and handler to mount.
+//
+// The handler is wrapped so the verified client identity reaches the stream:
+// connect exposes the peer's address but not its certificate.
 func (s *Service) Handler() (string, http.Handler) {
-	return llmgwv1connect.NewAgentServiceHandler(s)
+	path, h := llmgwv1connect.NewAgentServiceHandler(s)
+	return path, AuthMiddleware(h)
 }
 
 // Connect serves one agent's stream for the lifetime of that agent.
@@ -85,6 +101,24 @@ func (s *Service) Connect(
 		return connect.NewError(connect.CodeInvalidArgument, errors.New("node_id is required"))
 	}
 
+	// The certificate decides who this is, not the message.
+	//
+	// Without this check an agent could register as any node in the fleet and
+	// receive that node's shards — which is the same exposure the stream had
+	// before TLS, just with encryption on top. mTLS is only worth having if the
+	// identity it proves is the one actually used.
+	if peer := peerFrom(ctx); peer.Verified {
+		if peer.CommonName != nodeID {
+			s.log.Warn("rejecting an agent claiming a node id its certificate does not authorize",
+				"certificate_cn", peer.CommonName, "claimed", nodeID)
+			return connect.NewError(connect.CodePermissionDenied,
+				fmt.Errorf("certificate authorizes node %q, not %q", peer.CommonName, nodeID))
+		}
+	} else if s.requireIdentity {
+		return connect.NewError(connect.CodeUnauthenticated,
+			errors.New("no verified client certificate"))
+	}
+
 	node := NodeFromRegister(reg)
 	s.inv.Upsert(node)
 	s.log.Info("agent registered",

internal/tlsconf/ca.go

@@ -0,0 +1,223 @@
+package tlsconf
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"math/big"
+	"net"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+// CA is a certificate authority for one llmgw cluster.
+//
+// It exists because the alternative is worse. Public authorities no longer
+// issue certificates carrying the clientAuth usage, so a Let's Encrypt
+// certificate cannot authenticate an agent however convenient it would be. And
+// a self-signed certificate per node would mean every machine trusting every
+// other machine's certificate individually, which stops being manageable the
+// moment the fleet changes.
+//
+// A private CA is the ordinary answer: one root everyone trusts, leaves issued
+// from it, and identity carried in the subject where the control plane can read
+// it. Nothing outside llmgw ever sees these certificates — the client-facing
+// API is fronted by a publicly trusted certificate instead.
+type CA struct {
+	Cert *x509.Certificate
+	Key  *ecdsa.PrivateKey
+	// PEM is the encoded certificate, which is what goes on every machine.
+	PEM []byte
+}
+
+// CAValidity is how long a new authority is good for.
+//
+// Long, deliberately. Rotating a CA means touching every machine at once, so it
+// should be a planned event rather than something that surprises a cluster.
+const CAValidity = 10 * 365 * 24 * time.Hour
+
+// LeafValidity is how long an issued certificate is good for.
+//
+// A year rather than 90 days: there is no ACME here, renewal is a manual step,
+// and a short lifetime would turn that into recurring unplanned work. Long
+// enough to be calm, short enough that a leaked key is not permanent.
+const LeafValidity = 365 * 24 * time.Hour
+
+// NewCA creates an authority.
+func NewCA(commonName string) (*CA, error) {
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, err
+	}
+	serial, err := randomSerial()
+	if err != nil {
+		return nil, err
+	}
+
+	tmpl := &x509.Certificate{
+		SerialNumber:          serial,
+		Subject:               pkix.Name{CommonName: commonName},
+		NotBefore:             time.Now().Add(-time.Hour), // tolerate clock skew across the fleet
+		NotAfter:              time.Now().Add(CAValidity),
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		MaxPathLenZero:        true, // leaves only; this CA never signs another CA
+	}
+
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	if err != nil {
+		return nil, err
+	}
+	cert, err := x509.ParseCertificate(der)
+	if err != nil {
+		return nil, err
+	}
+	return &CA{Cert: cert, Key: key, PEM: pemEncode("CERTIFICATE", der)}, nil
+}
+
+// LoadCA reads an authority from disk.
+func LoadCA(certPath, keyPath string) (*CA, error) {
+	certPEM, err := os.ReadFile(certPath)
+	if err != nil {
+		return nil, err
+	}
+	block, _ := pem.Decode(certPEM)
+	if block == nil {
+		return nil, fmt.Errorf("%s is not PEM", certPath)
+	}
+	cert, err := x509.ParseCertificate(block.Bytes)
+	if err != nil {
+		return nil, err
+	}
+
+	keyPEM, err := os.ReadFile(keyPath)
+	if err != nil {
+		return nil, err
+	}
+	kb, _ := pem.Decode(keyPEM)
+	if kb == nil {
+		return nil, fmt.Errorf("%s is not PEM", keyPath)
+	}
+	key, err := x509.ParseECPrivateKey(kb.Bytes)
+	if err != nil {
+		return nil, err
+	}
+	return &CA{Cert: cert, Key: key, PEM: certPEM}, nil
+}
+
+// Leaf is an issued certificate and its key, both PEM encoded.
+type Leaf struct {
+	CertPEM []byte
+	KeyPEM  []byte
+}
+
+// IssueServer issues a certificate for something agents connect to.
+//
+// The hosts are what matter here, not the common name: since Go 1.15 hostname
+// verification reads subject alternative names only and ignores the common name
+// entirely. A certificate without the address agents dial — including the bare
+// IP, if that is what they dial — is rejected no matter what its subject says.
+func (ca *CA) IssueServer(commonName string, hosts []string) (Leaf, error) {
+	if len(hosts) == 0 {
+		return Leaf{}, fmt.Errorf("a server certificate needs at least one host; " +
+			"without a matching SAN every client rejects it")
+	}
+	return ca.issue(commonName, hosts, x509.ExtKeyUsageServerAuth)
+}
+
+// IssueClient issues a certificate identifying one node.
+//
+// The common name is the node id, and the control plane refuses any agent whose
+// claimed id does not match it. That is the whole point of the exercise: with
+// it, a node's identity is something it proves rather than something it picks.
+func (ca *CA) IssueClient(nodeID string) (Leaf, error) {
+	if nodeID == "" {
+		return Leaf{}, fmt.Errorf("a client certificate needs a node id as its common name")
+	}
+	return ca.issue(nodeID, nil, x509.ExtKeyUsageClientAuth)
+}
+
+func (ca *CA) issue(commonName string, hosts []string, usage x509.ExtKeyUsage) (Leaf, error) {
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return Leaf{}, err
+	}
+	serial, err := randomSerial()
+	if err != nil {
+		return Leaf{}, err
+	}
+
+	tmpl := &x509.Certificate{
+		SerialNumber: serial,
+		Subject:      pkix.Name{CommonName: commonName},
+		NotBefore:    time.Now().Add(-time.Hour),
+		NotAfter:     time.Now().Add(LeafValidity),
+		KeyUsage:     x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:  []x509.ExtKeyUsage{usage},
+	}
+	for _, h := range hosts {
+		if ip := net.ParseIP(h); ip != nil {
+			tmpl.IPAddresses = append(tmpl.IPAddresses, ip)
+		} else {
+			tmpl.DNSNames = append(tmpl.DNSNames, h)
+		}
+	}
+
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.Cert, &key.PublicKey, ca.Key)
+	if err != nil {
+		return Leaf{}, err
+	}
+	keyDER, err := x509.MarshalECPrivateKey(key)
+	if err != nil {
+		return Leaf{}, err
+	}
+	return Leaf{
+		CertPEM: pemEncode("CERTIFICATE", der),
+		KeyPEM:  pemEncode("EC PRIVATE KEY", keyDER),
+	}, nil
+}
+
+// WriteCA writes the authority to a directory: the certificate world-readable,
+// the key not.
+func (ca *CA) Write(dir string) error {
+	keyDER, err := x509.MarshalECPrivateKey(ca.Key)
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return err
+	}
+	if err := os.WriteFile(filepath.Join(dir, "ca.pem"), ca.PEM, 0o644); err != nil {
+		return err
+	}
+	// The CA key signs every identity in the cluster. It belongs on whichever
+	// machine issues certificates and nowhere else.
+	return os.WriteFile(filepath.Join(dir, "ca-key.pem"), pemEncode("EC PRIVATE KEY", keyDER), 0o600)
+}
+
+// Write saves a leaf under dir as <name>.pem and <name>-key.pem.
+func (l Leaf) Write(dir, name string) error {
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return err
+	}
+	if err := os.WriteFile(filepath.Join(dir, name+".pem"), l.CertPEM, 0o644); err != nil {
+		return err
+	}
+	return os.WriteFile(filepath.Join(dir, name+"-key.pem"), l.KeyPEM, 0o600)
+}
+
+func pemEncode(kind string, der []byte) []byte {
+	return pem.EncodeToMemory(&pem.Block{Type: kind, Bytes: der})
+}
+
+func randomSerial() (*big.Int, error) {
+	// 128 bits, as the CA/Browser Forum requires and every sane issuer does:
+	// a predictable serial is one of the ingredients of a chosen-prefix attack.
+	return rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+}

internal/tlsconf/ca_test.go

@@ -0,0 +1,268 @@
+package tlsconf
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"testing"
+
+	"golang.org/x/net/http2"
+)
+
+func newCA(t *testing.T) *CA {
+	t.Helper()
+	ca, err := NewCA("llmgw test CA")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return ca
+}
+
+// writeCluster issues a server and client certificate and returns the configs
+// that would load them, exactly as the real components do.
+func writeCluster(t *testing.T, ca *CA, serverHosts []string, nodeID string) (server, client Config) {
+	t.Helper()
+	dir := t.TempDir()
+
+	if err := ca.Write(dir); err != nil {
+		t.Fatal(err)
+	}
+	srv, err := ca.IssueServer("llmgw-ctl", serverHosts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := srv.Write(dir, "server"); err != nil {
+		t.Fatal(err)
+	}
+	cli, err := ca.IssueClient(nodeID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := cli.Write(dir, "client"); err != nil {
+		t.Fatal(err)
+	}
+
+	caPath := filepath.Join(dir, "ca.pem")
+	return Config{CA: caPath, Cert: filepath.Join(dir, "server.pem"), Key: filepath.Join(dir, "server-key.pem")},
+		Config{CA: caPath, Cert: filepath.Join(dir, "client.pem"), Key: filepath.Join(dir, "client-key.pem")}
+}
+
+func TestMutualAuthenticationSucceeds(t *testing.T) {
+	ca := newCA(t)
+	serverCfg, clientCfg := writeCluster(t, ca, []string{"127.0.0.1"}, "llmgw-node01")
+
+	stls, err := serverCfg.ServerConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// The identity the handler sees is what the control plane keys on.
+		if len(r.TLS.PeerCertificates) == 0 {
+			t.Error("no client certificate reached the handler")
+			return
+		}
+		_, _ = w.Write([]byte(r.TLS.PeerCertificates[0].Subject.CommonName))
+	}))
+	srv.TLS = stls
+	srv.StartTLS()
+	defer srv.Close()
+
+	ctls, err := clientCfg.ClientConfig("127.0.0.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := &http.Client{Transport: &http2.Transport{TLSClientConfig: ctls}}
+
+	resp, err := c.Get(srv.URL) //nolint:noctx // test
+	if err != nil {
+		t.Fatalf("mutual authentication failed: %v", err)
+	}
+	defer resp.Body.Close()
+
+	buf := make([]byte, 64)
+	n, _ := resp.Body.Read(buf)
+	if got := string(buf[:n]); got != "llmgw-node01" {
+		t.Errorf("the handler saw common name %q", got)
+	}
+}
+
+func TestServerRejectsClientWithoutCertificate(t *testing.T) {
+	// An agent that cannot prove who it is must not reach the control plane at
+	// all; accepting it unauthenticated would leave identity as forgeable as it
+	// was before TLS.
+	ca := newCA(t)
+	serverCfg, _ := writeCluster(t, ca, []string{"127.0.0.1"}, "llmgw-node01")
+
+	stls, _ := serverCfg.ServerConfig()
+	srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	srv.TLS = stls
+	srv.StartTLS()
+	defer srv.Close()
+
+	pool := x509.NewCertPool()
+	pool.AppendCertsFromPEM(ca.PEM)
+	c := &http.Client{Transport: &http2.Transport{TLSClientConfig: &tls.Config{
+		RootCAs: pool, ServerName: "127.0.0.1", MinVersion: tls.VersionTLS13,
+		NextProtos: []string{"h2"},
+	}}}
+
+	if _, err := c.Get(srv.URL); err == nil { //nolint:noctx // test
+		t.Fatal("a client with no certificate was accepted")
+	}
+}
+
+func TestServerRejectsCertificateFromAnotherCA(t *testing.T) {
+	ca := newCA(t)
+	serverCfg, _ := writeCluster(t, ca, []string{"127.0.0.1"}, "llmgw-node01")
+
+	// An attacker's own authority, with a certificate naming a real node.
+	other := newCA(t)
+	_, foreign := writeCluster(t, other, []string{"127.0.0.1"}, "llmgw-node01")
+
+	stls, _ := serverCfg.ServerConfig()
+	srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	srv.TLS = stls
+	srv.StartTLS()
+	defer srv.Close()
+
+	ctls, _ := foreign.ClientConfig("127.0.0.1")
+	// It still needs to trust the server, or the failure would prove nothing.
+	pool := x509.NewCertPool()
+	pool.AppendCertsFromPEM(ca.PEM)
+	ctls.RootCAs = pool
+
+	c := &http.Client{Transport: &http2.Transport{TLSClientConfig: ctls}}
+	if _, err := c.Get(srv.URL); err == nil { //nolint:noctx // test
+		t.Fatal("a certificate from an unrelated authority was accepted")
+	}
+}
+
+func TestClientRejectsServerWithoutMatchingSAN(t *testing.T) {
+	// The trap worth having a test for: hostname verification reads subject
+	// alternative names and ignores the common name entirely, so a certificate
+	// issued only for a DNS name fails when agents dial an IP.
+	ca := newCA(t)
+	serverCfg, clientCfg := writeCluster(t, ca, []string{"llmgw-ctl.example"}, "llmgw-node01")
+
+	stls, _ := serverCfg.ServerConfig()
+	srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	srv.TLS = stls
+	srv.StartTLS()
+	defer srv.Close()
+
+	ctls, _ := clientCfg.ClientConfig("127.0.0.1")
+	c := &http.Client{Transport: &http2.Transport{TLSClientConfig: ctls}}
+
+	_, err := c.Get(srv.URL) //nolint:noctx // test
+	if err == nil {
+		t.Fatal("a certificate with no matching SAN was accepted")
+	}
+}
+
+func TestIssuedCertificatesCarryTheRightUsage(t *testing.T) {
+	// The reason a publicly issued certificate cannot be used here: authorities
+	// stopped including clientAuth, and Go verifies client chains against it.
+	ca := newCA(t)
+
+	srv, err := ca.IssueServer("llmgw-ctl", []string{"10.0.0.140"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cli, err := ca.IssueClient("llmgw-node01")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	parse := func(l Leaf) *x509.Certificate {
+		t.Helper()
+		c, err := tls.X509KeyPair(l.CertPEM, l.KeyPEM)
+		if err != nil {
+			t.Fatal(err)
+		}
+		leaf, err := x509.ParseCertificate(c.Certificate[0])
+		if err != nil {
+			t.Fatal(err)
+		}
+		return leaf
+	}
+
+	s := parse(srv)
+	if len(s.ExtKeyUsage) != 1 || s.ExtKeyUsage[0] != x509.ExtKeyUsageServerAuth {
+		t.Errorf("server EKU=%v", s.ExtKeyUsage)
+	}
+	if len(s.IPAddresses) != 1 || !s.IPAddresses[0].Equal(net.ParseIP("10.0.0.140")) {
+		t.Errorf("the IP SAN is missing: %v", s.IPAddresses)
+	}
+
+	c := parse(cli)
+	if len(c.ExtKeyUsage) != 1 || c.ExtKeyUsage[0] != x509.ExtKeyUsageClientAuth {
+		t.Errorf("client EKU=%v", c.ExtKeyUsage)
+	}
+	if c.Subject.CommonName != "llmgw-node01" {
+		t.Errorf("the node id must be the common name, got %q", c.Subject.CommonName)
+	}
+}
+
+func TestServerCertificateRequiresHosts(t *testing.T) {
+	ca := newCA(t)
+	if _, err := ca.IssueServer("llmgw-ctl", nil); err == nil {
+		t.Error("a server certificate with no SAN was issued; every client would reject it")
+	}
+}
+
+func TestCARoundTripsThroughDisk(t *testing.T) {
+	ca := newCA(t)
+	dir := t.TempDir()
+	if err := ca.Write(dir); err != nil {
+		t.Fatal(err)
+	}
+
+	loaded, err := LoadCA(filepath.Join(dir, "ca.pem"), filepath.Join(dir, "ca-key.pem"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// A certificate issued by the reloaded authority must verify against the
+	// original, or issuing from a second machine would silently produce
+	// certificates nothing accepts.
+	leaf, err := loaded.IssueClient("llmgw-node02")
+	if err != nil {
+		t.Fatal(err)
+	}
+	pool := x509.NewCertPool()
+	pool.AppendCertsFromPEM(ca.PEM)
+
+	kp, _ := tls.X509KeyPair(leaf.CertPEM, leaf.KeyPEM)
+	parsed, _ := x509.ParseCertificate(kp.Certificate[0])
+	if _, err := parsed.Verify(x509.VerifyOptions{
+		Roots: pool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+	}); err != nil {
+		t.Errorf("a certificate from the reloaded CA does not verify: %v", err)
+	}
+}
+
+func TestConfigValidation(t *testing.T) {
+	if err := (Config{CA: "x", Cert: "y"}).Validate(); err == nil {
+		t.Error("a certificate without its key was accepted")
+	}
+	if err := (Config{Cert: "y", Key: "z"}).Validate(); err == nil {
+		t.Error("a certificate without a CA was accepted")
+	}
+	if err := (Config{Insecure: true, CA: "x"}).Validate(); err == nil {
+		t.Error("insecure alongside certificate paths was accepted")
+	}
+	if err := (Config{Insecure: true}).Validate(); err != nil {
+		t.Errorf("plain insecure was rejected: %v", err)
+	}
+	if err := (Config{}).Validate(); err != nil {
+		t.Errorf("an empty config was rejected: %v", err)
+	}
+}

internal/tlsconf/tlsconf.go

@@ -0,0 +1,112 @@
+// Package tlsconf loads the certificates the control plane and its agents
+// authenticate each other with (plan §5.6).
+//
+// Both directions are verified. The agent checking the server is the ordinary
+// half; the server checking the agent is the half that matters here, because
+// the client certificate is what makes a node's identity something it cannot
+// choose for itself.
+package tlsconf
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"os"
+)
+
+// Config names the files a component authenticates with.
+type Config struct {
+	// CA is the bundle used to verify the other side. In a real deployment this
+	// is whichever authority issued the fleet's certificates.
+	CA string `koanf:"ca"`
+	// Cert and Key are this component's own identity.
+	Cert string `koanf:"cert"`
+	Key  string `koanf:"key"`
+
+	// Insecure disables TLS entirely. It exists for local runs against
+	// sim-engines; anything using it must stay on a trusted network.
+	Insecure bool `koanf:"insecure"`
+}
+
+// Enabled reports whether TLS is configured.
+func (c Config) Enabled() bool { return !c.Insecure && c.CA != "" }
+
+// Validate rejects a half-configured setup, which would otherwise fail at the
+// first connection rather than at startup.
+func (c Config) Validate() error {
+	if c.Insecure {
+		if c.CA != "" || c.Cert != "" || c.Key != "" {
+			return errors.New("tls: insecure is set alongside certificate paths; pick one")
+		}
+		return nil
+	}
+	switch {
+	case c.CA == "" && c.Cert == "" && c.Key == "":
+		// Nothing configured at all. The caller decides whether that is
+		// acceptable; it is not this package's business to refuse.
+		return nil
+	case c.CA == "":
+		return errors.New("tls: ca is required")
+	case c.Cert == "" || c.Key == "":
+		return errors.New("tls: cert and key are both required")
+	}
+	return nil
+}
+
+// ServerConfig builds a TLS configuration that demands and verifies a client
+// certificate.
+//
+// RequireAndVerifyClientCert rather than VerifyClientCertIfGiven: an agent that
+// cannot prove who it is must not reach the control plane at all. Accepting it
+// unauthenticated would leave node identity exactly as forgeable as it was
+// without TLS.
+func (c Config) ServerConfig() (*tls.Config, error) {
+	cert, pool, err := c.load()
+	if err != nil {
+		return nil, err
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		ClientCAs:    pool,
+		ClientAuth:   tls.RequireAndVerifyClientCert,
+		MinVersion:   tls.VersionTLS13,
+		// connect's bidirectional streams need HTTP/2, which is negotiated
+		// through ALPN. Omitting this leaves the connection on HTTP/1.1 and the
+		// stream fails with an error that says nothing about why.
+		NextProtos: []string{"h2"},
+	}, nil
+}
+
+// ClientConfig builds a TLS configuration that presents this component's
+// certificate and verifies the server against the CA.
+func (c Config) ClientConfig(serverName string) (*tls.Config, error) {
+	cert, pool, err := c.load()
+	if err != nil {
+		return nil, err
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		RootCAs:      pool,
+		ServerName:   serverName,
+		MinVersion:   tls.VersionTLS13,
+		NextProtos:   []string{"h2"},
+	}, nil
+}
+
+func (c Config) load() (tls.Certificate, *x509.CertPool, error) {
+	cert, err := tls.LoadX509KeyPair(c.Cert, c.Key)
+	if err != nil {
+		return tls.Certificate{}, nil, fmt.Errorf("tls: loading %s: %w", c.Cert, err)
+	}
+
+	pem, err := os.ReadFile(c.CA)
+	if err != nil {
+		return tls.Certificate{}, nil, fmt.Errorf("tls: reading %s: %w", c.CA, err)
+	}
+	pool := x509.NewCertPool()
+	if !pool.AppendCertsFromPEM(pem) {
+		return tls.Certificate{}, nil, fmt.Errorf("tls: %s contains no certificates", c.CA)
+	}
+	return cert, pool, nil
+}