Revoke a removed node's certificate, and require --api-client with a mirror's mutual TLS

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 10:34:35 +02:00
1 parent fe5b7f88f0
commit 9b61bbc983
16 files changed
+268 -32

No files matched your search

+7 -1
View File
@@ -4,6 +4,9 @@ import (
"context"
"crypto/x509"
"net/http"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
// peerKey carries the verified client identity through to the stream handler.
@@ -21,6 +24,9 @@ type PeerIdentity struct {
// Verified is false when the connection carried no client certificate,
// which only happens when TLS is switched off.
Verified bool
// IssuedAt is when the certificate was issued, for revocation: one issued
// for a node before it was removed is refused.
IssuedAt time.Time
}
// WithPeerIdentity records a verified identity on the context.
@@ -53,5 +59,5 @@ func AuthMiddleware(next http.Handler) http.Handler {
}
func identityOf(cert *x509.Certificate) PeerIdentity {
return PeerIdentity{CommonName: cert.Subject.CommonName, Verified: true}
return PeerIdentity{CommonName: cert.Subject.CommonName, Verified: true, IssuedAt: tlsconf.IssuedAt(cert)}
}
@@ -0,0 +1,57 @@
package agentsvc
import (
"context"
"errors"
"testing"
"time"
"connectrpc.com/connect"
)
func TestARemovedNodesCertificateIsRefused(t *testing.T) {
removed := time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC)
s := newTestService()
s.requireIdentity = true
s.SetRevocations(func(_ context.Context, id string) (time.Time, error) {
if id == "gpu01" {
return removed, nil
}
return time.Time{}, nil
})
peer := func(cn string, issued time.Time) context.Context {
return WithPeerIdentity(context.Background(), PeerIdentity{CommonName: cn, Verified: true, IssuedAt: issued})
}
cases := []struct {
name string
ctx context.Context
node string
refuse connect.Code
}{
{"issued before the removal", peer("gpu01", removed.Add(-time.Minute)), "gpu01", connect.CodePermissionDenied},
{"issued at the removal", peer("gpu01", removed), "gpu01", connect.CodePermissionDenied},
{"issued after, by onboarding again", peer("gpu01", removed.Add(time.Second)), "gpu01", 0},
{"another node", peer("gpu02", removed.Add(-time.Hour)), "gpu02", 0},
{"another node's name", peer("gpu02", removed.Add(time.Hour)), "gpu01", connect.CodePermissionDenied},
{"no certificate", context.Background(), "gpu02", connect.CodeUnauthenticated},
}
for _, c := range cases {
err := s.admit(c.ctx, c.node)
if c.refuse == 0 {
if err != nil {
t.Errorf("%s: refused: %v", c.name, err)
}
continue
}
if connect.CodeOf(err) != c.refuse {
t.Errorf("%s: got %v, expected %v", c.name, err, c.refuse)
}
}
// Not knowing is not "not revoked".
s.SetRevocations(func(context.Context, string) (time.Time, error) { return time.Time{}, errors.New("store down") })
if err := s.admit(peer("gpu02", removed), "gpu02"); connect.CodeOf(err) != connect.CodeUnavailable {
t.Errorf("with the store down: %v, expected unavailable", err)
}
}
+46 -16
View File
@@ -49,6 +49,9 @@ type Service struct {
// requireIdentity refuses agents that present no certificate. It is off
// only for the insecure development path.
requireIdentity bool
// revokedAt is when a node was removed, whose certificates issued before
// then are refused; nil checks nothing.
revokedAt func(ctx context.Context, nodeID string) (time.Time, error)
mu sync.RWMutex
conns map[string]*conn
@@ -137,6 +140,47 @@ func New(inv *inventory.Store, status StatusSink, requireIdentity bool, log *slo
}
}
// admit decides whether the peer on ctx may be nodeID.
func (s *Service) admit(ctx context.Context, nodeID string) error {
// The certificate decides who this is, not the message.
//
// Without this check an agent could register as any node in the fleet and
// receive that node's shards — which is the same exposure the stream had
// before TLS, just with encryption on top. mTLS is only worth having if the
// identity it proves is the one actually used.
if peer := peerFrom(ctx); peer.Verified {
if peer.CommonName != nodeID {
s.log.Warn("rejecting an agent claiming a node id its certificate does not authorize",
"certificate_cn", peer.CommonName, "claimed", nodeID)
return connect.NewError(connect.CodePermissionDenied,
fmt.Errorf("certificate authorizes node %q, not %q", peer.CommonName, nodeID))
}
if s.revokedAt != nil {
at, err := s.revokedAt(ctx, nodeID)
if err != nil {
// Unknown is not "not revoked": the agent retries.
return connect.NewError(connect.CodeUnavailable, fmt.Errorf("checking revocation: %w", err))
}
if !at.IsZero() && !peer.IssuedAt.After(at) {
s.log.Warn("rejecting a removed node's certificate", "node", nodeID,
"issued", peer.IssuedAt, "removed", at)
return connect.NewError(connect.CodePermissionDenied, fmt.Errorf(
"node %q was removed from the cluster, and its certificate with it; onboard it again", nodeID))
}
}
} else if s.requireIdentity {
return connect.NewError(connect.CodeUnauthenticated,
errors.New("no verified client certificate"))
}
return nil
}
// SetRevocations has agents refused whose certificate was issued before
// their node was removed.
func (s *Service) SetRevocations(f func(ctx context.Context, nodeID string) (time.Time, error)) {
s.revokedAt = f
}
// Handler returns the HTTP path and handler to mount.
//
// The handler is wrapped so the verified client identity reaches the stream:
@@ -182,22 +226,8 @@ func (s *Service) Connect(
return connect.NewError(connect.CodeInvalidArgument, errors.New("node_id is required"))
}
// The certificate decides who this is, not the message.
//
// Without this check an agent could register as any node in the fleet and
// receive that node's shards — which is the same exposure the stream had
// before TLS, just with encryption on top. mTLS is only worth having if the
// identity it proves is the one actually used.
if peer := peerFrom(ctx); peer.Verified {
if peer.CommonName != nodeID {
s.log.Warn("rejecting an agent claiming a node id its certificate does not authorize",
"certificate_cn", peer.CommonName, "claimed", nodeID)
return connect.NewError(connect.CodePermissionDenied,
fmt.Errorf("certificate authorizes node %q, not %q", peer.CommonName, nodeID))
}
} else if s.requireIdentity {
return connect.NewError(connect.CodeUnauthenticated,
errors.New("no verified client certificate"))
if err := s.admit(ctx, nodeID); err != nil {
return err
}
node := NodeFromRegister(reg)