diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 16d8750..7ab4e4f 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -208,7 +208,7 @@ func startWith(t *testing.T, log *slog.Logger, cfgFns ...func(*Config)) *harness inv := inventory.NewStore() rec := newRecorder() - svc := agentsvc.New(inv, rec, log) + svc := agentsvc.New(inv, rec, false, log) path, handler := svc.Handler() mux := http.NewServeMux() diff --git a/internal/cli/cert.go b/internal/cli/cert.go new file mode 100644 index 0000000..fa23275 --- /dev/null +++ b/internal/cli/cert.go @@ -0,0 +1,254 @@ +package cli + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" +) + +// defaultPKIDir is where the authority lives on the machine that issues +// certificates. Only that machine ever holds ca-key.pem; everything else gets +// ca.pem and its own leaf. +const defaultPKIDir = "/etc/llmgw/pki" + +func newCertCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "cert", + Short: "Issue the certificates the control plane and agents authenticate with", + Long: "llmgw's internal connections use mutual TLS, and a public authority cannot\n" + + "supply them: Let's Encrypt and its peers stopped issuing certificates\n" + + "carrying the clientAuth usage, so nothing they hand out can authenticate an\n" + + "agent. These commands run a small private authority instead.\n\n" + + "They work on files and talk to nothing, so they run wherever you keep the\n" + + "authority — not necessarily on a cluster machine.", + } + cmd.AddCommand(newInitCACmd(), newIssueServerCmd(), newIssueClientCmd(), newCertInspectCmd()) + return cmd +} + +func newInitCACmd() *cobra.Command { + var ( + dir string + cn string + force bool + ) + cmd := &cobra.Command{ + Use: "init-ca", + Short: "Create the cluster's certificate authority", + Long: "Writes ca.pem and ca-key.pem. Copy ca.pem to every machine; leave\n" + + "ca-key.pem here, because whoever holds it can mint any node's identity.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + keyPath := filepath.Join(dir, "ca-key.pem") + if _, err := os.Stat(keyPath); err == nil && !force { + // Overwriting the key orphans every certificate ever issued from + // it, across every machine, with no warning until connections + // start failing. + return fmt.Errorf("%s already exists; re-running would invalidate every "+ + "certificate issued from it (pass --force if that is what you want)", keyPath) + } + + ca, err := tlsconf.NewCA(cn) + if err != nil { + return err + } + if err := ca.Write(dir); err != nil { + return err + } + cmd.Printf("authority %q written to %s\n", cn, dir) + cmd.Printf(" ca.pem trust this everywhere (valid until %s)\n", + ca.Cert.NotAfter.Format(time.DateOnly)) + cmd.Printf(" ca-key.pem keep it here, mode 0600\n") + return nil + }, + } + cmd.Flags().StringVar(&dir, "dir", defaultPKIDir, "directory holding the authority") + cmd.Flags().StringVar(&cn, "cn", "llmgw cluster CA", "common name for the authority") + cmd.Flags().BoolVar(&force, "force", false, "replace an existing authority") + return cmd +} + +func newIssueServerCmd() *cobra.Command { + var ( + dir string + out string + hosts []string + name string + ) + cmd := &cobra.Command{ + Use: "issue-server ", + Short: "Issue the control plane's certificate", + Long: "Every address an agent might dial must appear in --host, including bare\n" + + "IP addresses. Hostname verification reads subject alternative names and\n" + + "ignores the common name entirely, so a certificate that names the host\n" + + "only in its subject is rejected by every client.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ca, err := loadCA(dir) + if err != nil { + return err + } + leaf, err := ca.IssueServer(args[0], hosts) + if err != nil { + return err + } + if name == "" { + name = "server" + } + if err := leaf.Write(out, name); err != nil { + return err + } + return describeIssued(cmd, out, name, hosts) + }, + } + cmd.Flags().StringVar(&dir, "dir", defaultPKIDir, "directory holding the authority") + cmd.Flags().StringVar(&out, "out", ".", "directory to write the certificate into") + cmd.Flags().StringVar(&name, "name", "server", "base filename for the certificate") + cmd.Flags().StringArrayVar(&hosts, "host", nil, "address agents dial; repeatable (IP or DNS name)") + return cmd +} + +func newIssueClientCmd() *cobra.Command { + var dir, out string + cmd := &cobra.Command{ + Use: "issue-client ", + Short: "Issue one agent's certificate", + Long: "The node id becomes the certificate's common name, and the control plane\n" + + "refuses any agent whose claimed id does not match it. That is what makes a\n" + + "node's identity something it proves rather than something it picks.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ca, err := loadCA(dir) + if err != nil { + return err + } + leaf, err := ca.IssueClient(args[0]) + if err != nil { + return err + } + if err := leaf.Write(out, args[0]); err != nil { + return err + } + return describeIssued(cmd, out, args[0], nil) + }, + } + cmd.Flags().StringVar(&dir, "dir", defaultPKIDir, "directory holding the authority") + cmd.Flags().StringVar(&out, "out", ".", "directory to write the certificate into") + return cmd +} + +func newCertInspectCmd() *cobra.Command { + return &cobra.Command{ + Use: "inspect ", + Short: "Print what a certificate actually says", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cert, err := readCert(args[0]) + if err != nil { + return err + } + + cmd.Printf("subject %s\n", cert.Subject) + cmd.Printf("issuer %s\n", cert.Issuer) + cmd.Printf("valid %s .. %s", cert.NotBefore.Format(time.DateOnly), + cert.NotAfter.Format(time.DateOnly)) + if d := time.Until(cert.NotAfter); d < 0 { + cmd.Printf(" (EXPIRED)") + } else { + cmd.Printf(" (%d days left)", int(d.Hours()/24)) + } + cmd.Println() + + if cert.IsCA { + cmd.Println("role certificate authority") + } + if u := usageNames(cert.ExtKeyUsage); u != "" { + cmd.Printf("usage %s\n", u) + } + if sans := sanNames(cert); sans != "" { + cmd.Printf("hosts %s\n", sans) + } else if !cert.IsCA && !hasUsage(cert, x509.ExtKeyUsageClientAuth) { + cmd.Println("hosts none — no client will accept this as a server certificate") + } + return nil + }, + } +} + +func loadCA(dir string) (*tlsconf.CA, error) { + ca, err := tlsconf.LoadCA(filepath.Join(dir, "ca.pem"), filepath.Join(dir, "ca-key.pem")) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("no authority in %s — run 'llmgw ctl cert init-ca --dir %s' first", dir, dir) + } + return nil, err + } + return ca, nil +} + +func describeIssued(cmd *cobra.Command, dir, name string, hosts []string) error { + cert, err := readCert(filepath.Join(dir, name+".pem")) + if err != nil { + return err + } + cmd.Printf("issued %s\n", cert.Subject.CommonName) + cmd.Printf(" %s\n", filepath.Join(dir, name+".pem")) + cmd.Printf(" %s\n", filepath.Join(dir, name+"-key.pem")) + if len(hosts) > 0 { + cmd.Printf(" hosts: %s\n", strings.Join(hosts, ", ")) + } + cmd.Printf(" expires %s\n", cert.NotAfter.Format(time.DateOnly)) + return nil +} + +func readCert(path string) (*x509.Certificate, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + block, _ := pem.Decode(b) + if block == nil { + return nil, fmt.Errorf("%s is not PEM", path) + } + return x509.ParseCertificate(block.Bytes) +} + +func hasUsage(cert *x509.Certificate, want x509.ExtKeyUsage) bool { + for _, u := range cert.ExtKeyUsage { + if u == want { + return true + } + } + return false +} + +func usageNames(usages []x509.ExtKeyUsage) string { + var out []string + for _, u := range usages { + switch u { + case x509.ExtKeyUsageServerAuth: + out = append(out, "serverAuth (identifies the control plane)") + case x509.ExtKeyUsageClientAuth: + out = append(out, "clientAuth (identifies an agent)") + default: + out = append(out, fmt.Sprintf("%v", u)) + } + } + return strings.Join(out, ", ") +} + +func sanNames(cert *x509.Certificate) string { + names := append([]string{}, cert.DNSNames...) + for _, ip := range cert.IPAddresses { + names = append(names, ip.String()) + } + return strings.Join(names, ", ") +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 55d3b0b..c4f1f7a 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -106,6 +106,9 @@ func newCtlCmd() *cobra.Command { "Allow placement on a node again"), newNodeStateCmd(client, "drain", "draining", "Move work off a node, one replica at a time"), + // cert works on files rather than the admin API, so --server does not + // apply to it; it lives here because this is where operators look. + newCertCmd(), ) return cmd } diff --git a/internal/cli/server.go b/internal/cli/server.go index b40a41f..322ccd0 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -93,7 +93,7 @@ func newServerCmd() *cobra.Command { return err } - ctl, svc, err := startControlPlane(ctx, cfg, cat, rt, log) + ctl, svc, err := startControlPlane(ctx, cfg, cat, rt, false, log) if err != nil { return err } @@ -196,7 +196,7 @@ func newServerCmd() *cobra.Command { // it holds the open streams, so it is the only thing that can deliver a command // to a node. func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Catalog, - sink control.EndpointSink, log *slog.Logger, + sink control.EndpointSink, requireIdentity bool, log *slog.Logger, ) (*control.Controller, *agentsvc.Service, error) { ccfg := control.DefaultConfig() ccfg.Reconciler = reconciler.Config{ @@ -217,7 +217,7 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Cata // the service once it exists. fwd := &dispatcherRef{} ctl := control.New(ccfg, cat, fwd, sink, log) - svc := agentsvc.New(ctl.Inventory(), ctl.Reconciler(), log) + svc := agentsvc.New(ctl.Inventory(), ctl.Reconciler(), requireIdentity, log) fwd.set(svc) ctl.Start(ctx) diff --git a/internal/control/agentsvc/identity.go b/internal/control/agentsvc/identity.go new file mode 100644 index 0000000..ee626b4 --- /dev/null +++ b/internal/control/agentsvc/identity.go @@ -0,0 +1,57 @@ +package agentsvc + +import ( + "context" + "crypto/x509" + "net/http" +) + +// peerKey carries the verified client identity through to the stream handler. +// +// connect's BidiStream exposes the peer's address but not its certificate, so +// the identity is lifted out of the TLS state by middleware and put on the +// context before the connect handler ever sees the request. +type peerKey struct{} + +// PeerIdentity is who the other end proved itself to be. +type PeerIdentity struct { + // CommonName is the certificate's subject CN, which is the node id a node + // is allowed to claim. + CommonName string + // Verified is false when the connection carried no client certificate, + // which only happens when TLS is switched off. + Verified bool +} + +// WithPeerIdentity records a verified identity on the context. +func WithPeerIdentity(ctx context.Context, id PeerIdentity) context.Context { + return context.WithValue(ctx, peerKey{}, id) +} + +// peerFrom reads the identity a request was authenticated with. +func peerFrom(ctx context.Context) PeerIdentity { + id, _ := ctx.Value(peerKey{}).(PeerIdentity) + return id +} + +// AuthMiddleware lifts the client certificate's common name onto the request +// context. +// +// The certificate chain has already been verified by the TLS handshake — this +// only makes the result reachable from the handler. Without a certificate the +// identity is marked unverified and the handler decides what to do, which keeps +// the insecure development path working without a second code path through the +// service itself. +func AuthMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var id PeerIdentity + if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 { + id = identityOf(r.TLS.PeerCertificates[0]) + } + next.ServeHTTP(w, r.WithContext(WithPeerIdentity(r.Context(), id))) + }) +} + +func identityOf(cert *x509.Certificate) PeerIdentity { + return PeerIdentity{CommonName: cert.Subject.CommonName, Verified: true} +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 9f96c6d..0761d2d 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -34,6 +34,10 @@ type Service struct { status StatusSink log *slog.Logger + // requireIdentity refuses agents that present no certificate. It is off + // only for the insecure development path. + requireIdentity bool + mu sync.RWMutex conns map[string]*conn } @@ -52,16 +56,28 @@ type conn struct { func (c *conn) close() { c.once.Do(func() { close(c.closed) }) } // New builds the service. -func New(inv *inventory.Store, status StatusSink, log *slog.Logger) *Service { +// +// requireIdentity refuses any agent that has not proved who it is. It should be +// true whenever TLS is configured, and is false only for local runs without +// certificates. +func New(inv *inventory.Store, status StatusSink, requireIdentity bool, log *slog.Logger) *Service { if log == nil { log = slog.Default() } - return &Service{inv: inv, status: status, log: log, conns: map[string]*conn{}} + return &Service{ + inv: inv, status: status, log: log, + requireIdentity: requireIdentity, + conns: map[string]*conn{}, + } } // Handler returns the HTTP path and handler to mount. +// +// The handler is wrapped so the verified client identity reaches the stream: +// connect exposes the peer's address but not its certificate. func (s *Service) Handler() (string, http.Handler) { - return llmgwv1connect.NewAgentServiceHandler(s) + path, h := llmgwv1connect.NewAgentServiceHandler(s) + return path, AuthMiddleware(h) } // Connect serves one agent's stream for the lifetime of that agent. @@ -85,6 +101,24 @@ func (s *Service) Connect( return connect.NewError(connect.CodeInvalidArgument, errors.New("node_id is required")) } + // The certificate decides who this is, not the message. + // + // Without this check an agent could register as any node in the fleet and + // receive that node's shards — which is the same exposure the stream had + // before TLS, just with encryption on top. mTLS is only worth having if the + // identity it proves is the one actually used. + if peer := peerFrom(ctx); peer.Verified { + if peer.CommonName != nodeID { + s.log.Warn("rejecting an agent claiming a node id its certificate does not authorize", + "certificate_cn", peer.CommonName, "claimed", nodeID) + return connect.NewError(connect.CodePermissionDenied, + fmt.Errorf("certificate authorizes node %q, not %q", peer.CommonName, nodeID)) + } + } else if s.requireIdentity { + return connect.NewError(connect.CodeUnauthenticated, + errors.New("no verified client certificate")) + } + node := NodeFromRegister(reg) s.inv.Upsert(node) s.log.Info("agent registered", diff --git a/internal/tlsconf/ca.go b/internal/tlsconf/ca.go new file mode 100644 index 0000000..932bf88 --- /dev/null +++ b/internal/tlsconf/ca.go @@ -0,0 +1,223 @@ +package tlsconf + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "path/filepath" + "time" +) + +// CA is a certificate authority for one llmgw cluster. +// +// It exists because the alternative is worse. Public authorities no longer +// issue certificates carrying the clientAuth usage, so a Let's Encrypt +// certificate cannot authenticate an agent however convenient it would be. And +// a self-signed certificate per node would mean every machine trusting every +// other machine's certificate individually, which stops being manageable the +// moment the fleet changes. +// +// A private CA is the ordinary answer: one root everyone trusts, leaves issued +// from it, and identity carried in the subject where the control plane can read +// it. Nothing outside llmgw ever sees these certificates — the client-facing +// API is fronted by a publicly trusted certificate instead. +type CA struct { + Cert *x509.Certificate + Key *ecdsa.PrivateKey + // PEM is the encoded certificate, which is what goes on every machine. + PEM []byte +} + +// CAValidity is how long a new authority is good for. +// +// Long, deliberately. Rotating a CA means touching every machine at once, so it +// should be a planned event rather than something that surprises a cluster. +const CAValidity = 10 * 365 * 24 * time.Hour + +// LeafValidity is how long an issued certificate is good for. +// +// A year rather than 90 days: there is no ACME here, renewal is a manual step, +// and a short lifetime would turn that into recurring unplanned work. Long +// enough to be calm, short enough that a leaked key is not permanent. +const LeafValidity = 365 * 24 * time.Hour + +// NewCA creates an authority. +func NewCA(commonName string) (*CA, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + serial, err := randomSerial() + if err != nil { + return nil, err + } + + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: commonName}, + NotBefore: time.Now().Add(-time.Hour), // tolerate clock skew across the fleet + NotAfter: time.Now().Add(CAValidity), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + IsCA: true, + MaxPathLenZero: true, // leaves only; this CA never signs another CA + } + + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + return nil, err + } + cert, err := x509.ParseCertificate(der) + if err != nil { + return nil, err + } + return &CA{Cert: cert, Key: key, PEM: pemEncode("CERTIFICATE", der)}, nil +} + +// LoadCA reads an authority from disk. +func LoadCA(certPath, keyPath string) (*CA, error) { + certPEM, err := os.ReadFile(certPath) + if err != nil { + return nil, err + } + block, _ := pem.Decode(certPEM) + if block == nil { + return nil, fmt.Errorf("%s is not PEM", certPath) + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, err + } + + keyPEM, err := os.ReadFile(keyPath) + if err != nil { + return nil, err + } + kb, _ := pem.Decode(keyPEM) + if kb == nil { + return nil, fmt.Errorf("%s is not PEM", keyPath) + } + key, err := x509.ParseECPrivateKey(kb.Bytes) + if err != nil { + return nil, err + } + return &CA{Cert: cert, Key: key, PEM: certPEM}, nil +} + +// Leaf is an issued certificate and its key, both PEM encoded. +type Leaf struct { + CertPEM []byte + KeyPEM []byte +} + +// IssueServer issues a certificate for something agents connect to. +// +// The hosts are what matter here, not the common name: since Go 1.15 hostname +// verification reads subject alternative names only and ignores the common name +// entirely. A certificate without the address agents dial — including the bare +// IP, if that is what they dial — is rejected no matter what its subject says. +func (ca *CA) IssueServer(commonName string, hosts []string) (Leaf, error) { + if len(hosts) == 0 { + return Leaf{}, fmt.Errorf("a server certificate needs at least one host; " + + "without a matching SAN every client rejects it") + } + return ca.issue(commonName, hosts, x509.ExtKeyUsageServerAuth) +} + +// IssueClient issues a certificate identifying one node. +// +// The common name is the node id, and the control plane refuses any agent whose +// claimed id does not match it. That is the whole point of the exercise: with +// it, a node's identity is something it proves rather than something it picks. +func (ca *CA) IssueClient(nodeID string) (Leaf, error) { + if nodeID == "" { + return Leaf{}, fmt.Errorf("a client certificate needs a node id as its common name") + } + return ca.issue(nodeID, nil, x509.ExtKeyUsageClientAuth) +} + +func (ca *CA) issue(commonName string, hosts []string, usage x509.ExtKeyUsage) (Leaf, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return Leaf{}, err + } + serial, err := randomSerial() + if err != nil { + return Leaf{}, err + } + + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: commonName}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(LeafValidity), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{usage}, + } + for _, h := range hosts { + if ip := net.ParseIP(h); ip != nil { + tmpl.IPAddresses = append(tmpl.IPAddresses, ip) + } else { + tmpl.DNSNames = append(tmpl.DNSNames, h) + } + } + + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.Cert, &key.PublicKey, ca.Key) + if err != nil { + return Leaf{}, err + } + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return Leaf{}, err + } + return Leaf{ + CertPEM: pemEncode("CERTIFICATE", der), + KeyPEM: pemEncode("EC PRIVATE KEY", keyDER), + }, nil +} + +// WriteCA writes the authority to a directory: the certificate world-readable, +// the key not. +func (ca *CA) Write(dir string) error { + keyDER, err := x509.MarshalECPrivateKey(ca.Key) + if err != nil { + return err + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, "ca.pem"), ca.PEM, 0o644); err != nil { + return err + } + // The CA key signs every identity in the cluster. It belongs on whichever + // machine issues certificates and nowhere else. + return os.WriteFile(filepath.Join(dir, "ca-key.pem"), pemEncode("EC PRIVATE KEY", keyDER), 0o600) +} + +// Write saves a leaf under dir as .pem and -key.pem. +func (l Leaf) Write(dir, name string) error { + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, name+".pem"), l.CertPEM, 0o644); err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, name+"-key.pem"), l.KeyPEM, 0o600) +} + +func pemEncode(kind string, der []byte) []byte { + return pem.EncodeToMemory(&pem.Block{Type: kind, Bytes: der}) +} + +func randomSerial() (*big.Int, error) { + // 128 bits, as the CA/Browser Forum requires and every sane issuer does: + // a predictable serial is one of the ingredients of a chosen-prefix attack. + return rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) +} diff --git a/internal/tlsconf/ca_test.go b/internal/tlsconf/ca_test.go new file mode 100644 index 0000000..63df836 --- /dev/null +++ b/internal/tlsconf/ca_test.go @@ -0,0 +1,268 @@ +package tlsconf + +import ( + "crypto/tls" + "crypto/x509" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "golang.org/x/net/http2" +) + +func newCA(t *testing.T) *CA { + t.Helper() + ca, err := NewCA("llmgw test CA") + if err != nil { + t.Fatal(err) + } + return ca +} + +// writeCluster issues a server and client certificate and returns the configs +// that would load them, exactly as the real components do. +func writeCluster(t *testing.T, ca *CA, serverHosts []string, nodeID string) (server, client Config) { + t.Helper() + dir := t.TempDir() + + if err := ca.Write(dir); err != nil { + t.Fatal(err) + } + srv, err := ca.IssueServer("llmgw-ctl", serverHosts) + if err != nil { + t.Fatal(err) + } + if err := srv.Write(dir, "server"); err != nil { + t.Fatal(err) + } + cli, err := ca.IssueClient(nodeID) + if err != nil { + t.Fatal(err) + } + if err := cli.Write(dir, "client"); err != nil { + t.Fatal(err) + } + + caPath := filepath.Join(dir, "ca.pem") + return Config{CA: caPath, Cert: filepath.Join(dir, "server.pem"), Key: filepath.Join(dir, "server-key.pem")}, + Config{CA: caPath, Cert: filepath.Join(dir, "client.pem"), Key: filepath.Join(dir, "client-key.pem")} +} + +func TestMutualAuthenticationSucceeds(t *testing.T) { + ca := newCA(t) + serverCfg, clientCfg := writeCluster(t, ca, []string{"127.0.0.1"}, "llmgw-node01") + + stls, err := serverCfg.ServerConfig() + if err != nil { + t.Fatal(err) + } + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // The identity the handler sees is what the control plane keys on. + if len(r.TLS.PeerCertificates) == 0 { + t.Error("no client certificate reached the handler") + return + } + _, _ = w.Write([]byte(r.TLS.PeerCertificates[0].Subject.CommonName)) + })) + srv.TLS = stls + srv.StartTLS() + defer srv.Close() + + ctls, err := clientCfg.ClientConfig("127.0.0.1") + if err != nil { + t.Fatal(err) + } + c := &http.Client{Transport: &http2.Transport{TLSClientConfig: ctls}} + + resp, err := c.Get(srv.URL) //nolint:noctx // test + if err != nil { + t.Fatalf("mutual authentication failed: %v", err) + } + defer resp.Body.Close() + + buf := make([]byte, 64) + n, _ := resp.Body.Read(buf) + if got := string(buf[:n]); got != "llmgw-node01" { + t.Errorf("the handler saw common name %q", got) + } +} + +func TestServerRejectsClientWithoutCertificate(t *testing.T) { + // An agent that cannot prove who it is must not reach the control plane at + // all; accepting it unauthenticated would leave identity as forgeable as it + // was before TLS. + ca := newCA(t) + serverCfg, _ := writeCluster(t, ca, []string{"127.0.0.1"}, "llmgw-node01") + + stls, _ := serverCfg.ServerConfig() + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + srv.TLS = stls + srv.StartTLS() + defer srv.Close() + + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(ca.PEM) + c := &http.Client{Transport: &http2.Transport{TLSClientConfig: &tls.Config{ + RootCAs: pool, ServerName: "127.0.0.1", MinVersion: tls.VersionTLS13, + NextProtos: []string{"h2"}, + }}} + + if _, err := c.Get(srv.URL); err == nil { //nolint:noctx // test + t.Fatal("a client with no certificate was accepted") + } +} + +func TestServerRejectsCertificateFromAnotherCA(t *testing.T) { + ca := newCA(t) + serverCfg, _ := writeCluster(t, ca, []string{"127.0.0.1"}, "llmgw-node01") + + // An attacker's own authority, with a certificate naming a real node. + other := newCA(t) + _, foreign := writeCluster(t, other, []string{"127.0.0.1"}, "llmgw-node01") + + stls, _ := serverCfg.ServerConfig() + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + srv.TLS = stls + srv.StartTLS() + defer srv.Close() + + ctls, _ := foreign.ClientConfig("127.0.0.1") + // It still needs to trust the server, or the failure would prove nothing. + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(ca.PEM) + ctls.RootCAs = pool + + c := &http.Client{Transport: &http2.Transport{TLSClientConfig: ctls}} + if _, err := c.Get(srv.URL); err == nil { //nolint:noctx // test + t.Fatal("a certificate from an unrelated authority was accepted") + } +} + +func TestClientRejectsServerWithoutMatchingSAN(t *testing.T) { + // The trap worth having a test for: hostname verification reads subject + // alternative names and ignores the common name entirely, so a certificate + // issued only for a DNS name fails when agents dial an IP. + ca := newCA(t) + serverCfg, clientCfg := writeCluster(t, ca, []string{"llmgw-ctl.example"}, "llmgw-node01") + + stls, _ := serverCfg.ServerConfig() + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + srv.TLS = stls + srv.StartTLS() + defer srv.Close() + + ctls, _ := clientCfg.ClientConfig("127.0.0.1") + c := &http.Client{Transport: &http2.Transport{TLSClientConfig: ctls}} + + _, err := c.Get(srv.URL) //nolint:noctx // test + if err == nil { + t.Fatal("a certificate with no matching SAN was accepted") + } +} + +func TestIssuedCertificatesCarryTheRightUsage(t *testing.T) { + // The reason a publicly issued certificate cannot be used here: authorities + // stopped including clientAuth, and Go verifies client chains against it. + ca := newCA(t) + + srv, err := ca.IssueServer("llmgw-ctl", []string{"10.0.0.140"}) + if err != nil { + t.Fatal(err) + } + cli, err := ca.IssueClient("llmgw-node01") + if err != nil { + t.Fatal(err) + } + + parse := func(l Leaf) *x509.Certificate { + t.Helper() + c, err := tls.X509KeyPair(l.CertPEM, l.KeyPEM) + if err != nil { + t.Fatal(err) + } + leaf, err := x509.ParseCertificate(c.Certificate[0]) + if err != nil { + t.Fatal(err) + } + return leaf + } + + s := parse(srv) + if len(s.ExtKeyUsage) != 1 || s.ExtKeyUsage[0] != x509.ExtKeyUsageServerAuth { + t.Errorf("server EKU=%v", s.ExtKeyUsage) + } + if len(s.IPAddresses) != 1 || !s.IPAddresses[0].Equal(net.ParseIP("10.0.0.140")) { + t.Errorf("the IP SAN is missing: %v", s.IPAddresses) + } + + c := parse(cli) + if len(c.ExtKeyUsage) != 1 || c.ExtKeyUsage[0] != x509.ExtKeyUsageClientAuth { + t.Errorf("client EKU=%v", c.ExtKeyUsage) + } + if c.Subject.CommonName != "llmgw-node01" { + t.Errorf("the node id must be the common name, got %q", c.Subject.CommonName) + } +} + +func TestServerCertificateRequiresHosts(t *testing.T) { + ca := newCA(t) + if _, err := ca.IssueServer("llmgw-ctl", nil); err == nil { + t.Error("a server certificate with no SAN was issued; every client would reject it") + } +} + +func TestCARoundTripsThroughDisk(t *testing.T) { + ca := newCA(t) + dir := t.TempDir() + if err := ca.Write(dir); err != nil { + t.Fatal(err) + } + + loaded, err := LoadCA(filepath.Join(dir, "ca.pem"), filepath.Join(dir, "ca-key.pem")) + if err != nil { + t.Fatal(err) + } + // A certificate issued by the reloaded authority must verify against the + // original, or issuing from a second machine would silently produce + // certificates nothing accepts. + leaf, err := loaded.IssueClient("llmgw-node02") + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(ca.PEM) + + kp, _ := tls.X509KeyPair(leaf.CertPEM, leaf.KeyPEM) + parsed, _ := x509.ParseCertificate(kp.Certificate[0]) + if _, err := parsed.Verify(x509.VerifyOptions{ + Roots: pool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }); err != nil { + t.Errorf("a certificate from the reloaded CA does not verify: %v", err) + } +} + +func TestConfigValidation(t *testing.T) { + if err := (Config{CA: "x", Cert: "y"}).Validate(); err == nil { + t.Error("a certificate without its key was accepted") + } + if err := (Config{Cert: "y", Key: "z"}).Validate(); err == nil { + t.Error("a certificate without a CA was accepted") + } + if err := (Config{Insecure: true, CA: "x"}).Validate(); err == nil { + t.Error("insecure alongside certificate paths was accepted") + } + if err := (Config{Insecure: true}).Validate(); err != nil { + t.Errorf("plain insecure was rejected: %v", err) + } + if err := (Config{}).Validate(); err != nil { + t.Errorf("an empty config was rejected: %v", err) + } +} diff --git a/internal/tlsconf/tlsconf.go b/internal/tlsconf/tlsconf.go new file mode 100644 index 0000000..a1ac97f --- /dev/null +++ b/internal/tlsconf/tlsconf.go @@ -0,0 +1,112 @@ +// Package tlsconf loads the certificates the control plane and its agents +// authenticate each other with (plan §5.6). +// +// Both directions are verified. The agent checking the server is the ordinary +// half; the server checking the agent is the half that matters here, because +// the client certificate is what makes a node's identity something it cannot +// choose for itself. +package tlsconf + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "os" +) + +// Config names the files a component authenticates with. +type Config struct { + // CA is the bundle used to verify the other side. In a real deployment this + // is whichever authority issued the fleet's certificates. + CA string `koanf:"ca"` + // Cert and Key are this component's own identity. + Cert string `koanf:"cert"` + Key string `koanf:"key"` + + // Insecure disables TLS entirely. It exists for local runs against + // sim-engines; anything using it must stay on a trusted network. + Insecure bool `koanf:"insecure"` +} + +// Enabled reports whether TLS is configured. +func (c Config) Enabled() bool { return !c.Insecure && c.CA != "" } + +// Validate rejects a half-configured setup, which would otherwise fail at the +// first connection rather than at startup. +func (c Config) Validate() error { + if c.Insecure { + if c.CA != "" || c.Cert != "" || c.Key != "" { + return errors.New("tls: insecure is set alongside certificate paths; pick one") + } + return nil + } + switch { + case c.CA == "" && c.Cert == "" && c.Key == "": + // Nothing configured at all. The caller decides whether that is + // acceptable; it is not this package's business to refuse. + return nil + case c.CA == "": + return errors.New("tls: ca is required") + case c.Cert == "" || c.Key == "": + return errors.New("tls: cert and key are both required") + } + return nil +} + +// ServerConfig builds a TLS configuration that demands and verifies a client +// certificate. +// +// RequireAndVerifyClientCert rather than VerifyClientCertIfGiven: an agent that +// cannot prove who it is must not reach the control plane at all. Accepting it +// unauthenticated would leave node identity exactly as forgeable as it was +// without TLS. +func (c Config) ServerConfig() (*tls.Config, error) { + cert, pool, err := c.load() + if err != nil { + return nil, err + } + return &tls.Config{ + Certificates: []tls.Certificate{cert}, + ClientCAs: pool, + ClientAuth: tls.RequireAndVerifyClientCert, + MinVersion: tls.VersionTLS13, + // connect's bidirectional streams need HTTP/2, which is negotiated + // through ALPN. Omitting this leaves the connection on HTTP/1.1 and the + // stream fails with an error that says nothing about why. + NextProtos: []string{"h2"}, + }, nil +} + +// ClientConfig builds a TLS configuration that presents this component's +// certificate and verifies the server against the CA. +func (c Config) ClientConfig(serverName string) (*tls.Config, error) { + cert, pool, err := c.load() + if err != nil { + return nil, err + } + return &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: pool, + ServerName: serverName, + MinVersion: tls.VersionTLS13, + NextProtos: []string{"h2"}, + }, nil +} + +func (c Config) load() (tls.Certificate, *x509.CertPool, error) { + cert, err := tls.LoadX509KeyPair(c.Cert, c.Key) + if err != nil { + return tls.Certificate{}, nil, fmt.Errorf("tls: loading %s: %w", c.Cert, err) + } + + pem, err := os.ReadFile(c.CA) + if err != nil { + return tls.Certificate{}, nil, fmt.Errorf("tls: reading %s: %w", c.CA, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return tls.Certificate{}, nil, fmt.Errorf("tls: %s contains no certificates", c.CA) + } + return cert, pool, nil +}