Revoke a removed node's certificate, and require --api-client with a mirror's mutual TLS
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
fe5b7f88f0
commit
9b61bbc983
16 files changed
+268
-32
No files matched your search
@@ -47,6 +47,8 @@ type API struct {
|
||||
serve *modelserve.Server
|
||||
// history is the Prometheus the console's charts read from.
|
||||
history History
|
||||
// revoke records a removed node, so its certificate is refused.
|
||||
revoke func(ctx context.Context, nodeID string, at time.Time) error
|
||||
|
||||
models Models
|
||||
capacity Capacity
|
||||
@@ -58,6 +60,11 @@ type API struct {
|
||||
// New builds the API over a controller.
|
||||
func New(ctl *control.Controller) *API { return &API{ctl: ctl} }
|
||||
|
||||
// SetRevoker has removing a node revoke its certificate.
|
||||
func (a *API) SetRevoker(f func(ctx context.Context, nodeID string, at time.Time) error) {
|
||||
a.revoke = f
|
||||
}
|
||||
|
||||
// Register mounts the endpoints on a mux under /api/v1.
|
||||
func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /api/v1/summary", a.summary)
|
||||
@@ -1098,6 +1105,14 @@ func (a *API) removeNode(w http.ResponseWriter, r *http.Request) {
|
||||
"%s is %s: stop its agent (systemctl stop llmgw-agent) and remove it once it is lost, a minute later", id, n.State))
|
||||
return
|
||||
}
|
||||
// Revoked first: a node removed but still holding a certificate the
|
||||
// agent port accepts could connect again and be given work.
|
||||
if a.revoke != nil {
|
||||
if err := a.revoke(r.Context(), id, time.Now()); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "revoking the node's certificate: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
inv.Remove(id)
|
||||
a.ctl.Reconciler().Notify()
|
||||
writeJSON(w, http.StatusOK, map[string]string{"id": id, "removed": "true"})
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -707,3 +708,30 @@ func TestOnlyANodeThatIsGoneCanBeRemoved(t *testing.T) {
|
||||
t.Errorf("removing it again gave %d, expected 404", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemovingANodeRevokesItsCertificate(t *testing.T) {
|
||||
a, ctl := setup(t, testModels(), gpuNode("n1", 8), gpuNode("n2", 8))
|
||||
var revoked []string
|
||||
a.SetRevoker(func(_ context.Context, id string, _ time.Time) error {
|
||||
revoked = append(revoked, id)
|
||||
return nil
|
||||
})
|
||||
ctl.Inventory().SetState("n1", inventory.Lost)
|
||||
if code := call(t, a, "DELETE", "/api/v1/nodes/n1", "", nil); code != 200 {
|
||||
t.Fatalf("a lost node gave %d, expected 200", code)
|
||||
}
|
||||
if !slices.Equal(revoked, []string{"n1"}) {
|
||||
t.Errorf("revoked %v, expected n1", revoked)
|
||||
}
|
||||
|
||||
// Not revoked, not removed: a node left listed is one the operator can
|
||||
// see and try again on.
|
||||
a.SetRevoker(func(context.Context, string, time.Time) error { return errors.New("store down") })
|
||||
ctl.Inventory().SetState("n2", inventory.Lost)
|
||||
if code := call(t, a, "DELETE", "/api/v1/nodes/n2", "", nil); code != 500 {
|
||||
t.Errorf("a failed revocation gave %d, expected 500", code)
|
||||
}
|
||||
if _, ok := ctl.Inventory().Get("n2"); !ok {
|
||||
t.Error("the node was removed though its certificate was not revoked")
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,9 @@ import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
// peerKey carries the verified client identity through to the stream handler.
|
||||
@@ -21,6 +24,9 @@ type PeerIdentity struct {
|
||||
// Verified is false when the connection carried no client certificate,
|
||||
// which only happens when TLS is switched off.
|
||||
Verified bool
|
||||
// IssuedAt is when the certificate was issued, for revocation: one issued
|
||||
// for a node before it was removed is refused.
|
||||
IssuedAt time.Time
|
||||
}
|
||||
|
||||
// WithPeerIdentity records a verified identity on the context.
|
||||
@@ -53,5 +59,5 @@ func AuthMiddleware(next http.Handler) http.Handler {
|
||||
}
|
||||
|
||||
func identityOf(cert *x509.Certificate) PeerIdentity {
|
||||
return PeerIdentity{CommonName: cert.Subject.CommonName, Verified: true}
|
||||
return PeerIdentity{CommonName: cert.Subject.CommonName, Verified: true, IssuedAt: tlsconf.IssuedAt(cert)}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package agentsvc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"connectrpc.com/connect"
|
||||
)
|
||||
|
||||
func TestARemovedNodesCertificateIsRefused(t *testing.T) {
|
||||
removed := time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC)
|
||||
s := newTestService()
|
||||
s.requireIdentity = true
|
||||
s.SetRevocations(func(_ context.Context, id string) (time.Time, error) {
|
||||
if id == "gpu01" {
|
||||
return removed, nil
|
||||
}
|
||||
return time.Time{}, nil
|
||||
})
|
||||
peer := func(cn string, issued time.Time) context.Context {
|
||||
return WithPeerIdentity(context.Background(), PeerIdentity{CommonName: cn, Verified: true, IssuedAt: issued})
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
ctx context.Context
|
||||
node string
|
||||
refuse connect.Code
|
||||
}{
|
||||
{"issued before the removal", peer("gpu01", removed.Add(-time.Minute)), "gpu01", connect.CodePermissionDenied},
|
||||
{"issued at the removal", peer("gpu01", removed), "gpu01", connect.CodePermissionDenied},
|
||||
{"issued after, by onboarding again", peer("gpu01", removed.Add(time.Second)), "gpu01", 0},
|
||||
{"another node", peer("gpu02", removed.Add(-time.Hour)), "gpu02", 0},
|
||||
{"another node's name", peer("gpu02", removed.Add(time.Hour)), "gpu01", connect.CodePermissionDenied},
|
||||
{"no certificate", context.Background(), "gpu02", connect.CodeUnauthenticated},
|
||||
}
|
||||
for _, c := range cases {
|
||||
err := s.admit(c.ctx, c.node)
|
||||
if c.refuse == 0 {
|
||||
if err != nil {
|
||||
t.Errorf("%s: refused: %v", c.name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if connect.CodeOf(err) != c.refuse {
|
||||
t.Errorf("%s: got %v, expected %v", c.name, err, c.refuse)
|
||||
}
|
||||
}
|
||||
|
||||
// Not knowing is not "not revoked".
|
||||
s.SetRevocations(func(context.Context, string) (time.Time, error) { return time.Time{}, errors.New("store down") })
|
||||
if err := s.admit(peer("gpu02", removed), "gpu02"); connect.CodeOf(err) != connect.CodeUnavailable {
|
||||
t.Errorf("with the store down: %v, expected unavailable", err)
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,9 @@ type Service struct {
|
||||
// requireIdentity refuses agents that present no certificate. It is off
|
||||
// only for the insecure development path.
|
||||
requireIdentity bool
|
||||
// revokedAt is when a node was removed, whose certificates issued before
|
||||
// then are refused; nil checks nothing.
|
||||
revokedAt func(ctx context.Context, nodeID string) (time.Time, error)
|
||||
|
||||
mu sync.RWMutex
|
||||
conns map[string]*conn
|
||||
@@ -137,6 +140,47 @@ func New(inv *inventory.Store, status StatusSink, requireIdentity bool, log *slo
|
||||
}
|
||||
}
|
||||
|
||||
// admit decides whether the peer on ctx may be nodeID.
|
||||
func (s *Service) admit(ctx context.Context, nodeID string) error {
|
||||
// The certificate decides who this is, not the message.
|
||||
//
|
||||
// Without this check an agent could register as any node in the fleet and
|
||||
// receive that node's shards — which is the same exposure the stream had
|
||||
// before TLS, just with encryption on top. mTLS is only worth having if the
|
||||
// identity it proves is the one actually used.
|
||||
if peer := peerFrom(ctx); peer.Verified {
|
||||
if peer.CommonName != nodeID {
|
||||
s.log.Warn("rejecting an agent claiming a node id its certificate does not authorize",
|
||||
"certificate_cn", peer.CommonName, "claimed", nodeID)
|
||||
return connect.NewError(connect.CodePermissionDenied,
|
||||
fmt.Errorf("certificate authorizes node %q, not %q", peer.CommonName, nodeID))
|
||||
}
|
||||
if s.revokedAt != nil {
|
||||
at, err := s.revokedAt(ctx, nodeID)
|
||||
if err != nil {
|
||||
// Unknown is not "not revoked": the agent retries.
|
||||
return connect.NewError(connect.CodeUnavailable, fmt.Errorf("checking revocation: %w", err))
|
||||
}
|
||||
if !at.IsZero() && !peer.IssuedAt.After(at) {
|
||||
s.log.Warn("rejecting a removed node's certificate", "node", nodeID,
|
||||
"issued", peer.IssuedAt, "removed", at)
|
||||
return connect.NewError(connect.CodePermissionDenied, fmt.Errorf(
|
||||
"node %q was removed from the cluster, and its certificate with it; onboard it again", nodeID))
|
||||
}
|
||||
}
|
||||
} else if s.requireIdentity {
|
||||
return connect.NewError(connect.CodeUnauthenticated,
|
||||
errors.New("no verified client certificate"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetRevocations has agents refused whose certificate was issued before
|
||||
// their node was removed.
|
||||
func (s *Service) SetRevocations(f func(ctx context.Context, nodeID string) (time.Time, error)) {
|
||||
s.revokedAt = f
|
||||
}
|
||||
|
||||
// Handler returns the HTTP path and handler to mount.
|
||||
//
|
||||
// The handler is wrapped so the verified client identity reaches the stream:
|
||||
@@ -182,22 +226,8 @@ func (s *Service) Connect(
|
||||
return connect.NewError(connect.CodeInvalidArgument, errors.New("node_id is required"))
|
||||
}
|
||||
|
||||
// The certificate decides who this is, not the message.
|
||||
//
|
||||
// Without this check an agent could register as any node in the fleet and
|
||||
// receive that node's shards — which is the same exposure the stream had
|
||||
// before TLS, just with encryption on top. mTLS is only worth having if the
|
||||
// identity it proves is the one actually used.
|
||||
if peer := peerFrom(ctx); peer.Verified {
|
||||
if peer.CommonName != nodeID {
|
||||
s.log.Warn("rejecting an agent claiming a node id its certificate does not authorize",
|
||||
"certificate_cn", peer.CommonName, "claimed", nodeID)
|
||||
return connect.NewError(connect.CodePermissionDenied,
|
||||
fmt.Errorf("certificate authorizes node %q, not %q", peer.CommonName, nodeID))
|
||||
}
|
||||
} else if s.requireIdentity {
|
||||
return connect.NewError(connect.CodeUnauthenticated,
|
||||
errors.New("no verified client certificate"))
|
||||
if err := s.admit(ctx, nodeID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
node := NodeFromRegister(reg)
|
||||
|
||||
Reference in new issue
Block a user