Revoke a removed node's certificate, and require --api-client with a mirror's mutual TLS

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

internal/control/adminapi/api.go

@@ -47,6 +47,8 @@ type API struct {
 	serve *modelserve.Server
 	// history is the Prometheus the console's charts read from.
 	history History
+	// revoke records a removed node, so its certificate is refused.
+	revoke func(ctx context.Context, nodeID string, at time.Time) error
 
 	models   Models
 	capacity Capacity
@@ -58,6 +60,11 @@ type API struct {
 // New builds the API over a controller.
 func New(ctl *control.Controller) *API { return &API{ctl: ctl} }
 
+// SetRevoker has removing a node revoke its certificate.
+func (a *API) SetRevoker(f func(ctx context.Context, nodeID string, at time.Time) error) {
+	a.revoke = f
+}
+
 // Register mounts the endpoints on a mux under /api/v1.
 func (a *API) Register(mux *http.ServeMux) {
 	mux.HandleFunc("GET /api/v1/summary", a.summary)
@@ -1098,6 +1105,14 @@ func (a *API) removeNode(w http.ResponseWriter, r *http.Request) {
 			"%s is %s: stop its agent (systemctl stop llmgw-agent) and remove it once it is lost, a minute later", id, n.State))
 		return
 	}
+	// Revoked first: a node removed but still holding a certificate the
+	// agent port accepts could connect again and be given work.
+	if a.revoke != nil {
+		if err := a.revoke(r.Context(), id, time.Now()); err != nil {
+			writeError(w, http.StatusInternalServerError, "revoking the node's certificate: "+err.Error())
+			return
+		}
+	}
 	inv.Remove(id)
 	a.ctl.Reconciler().Notify()
 	writeJSON(w, http.StatusOK, map[string]string{"id": id, "removed": "true"})

internal/control/adminapi/api_test.go

@@ -7,6 +7,7 @@ import (
 	"log/slog"
 	"net/http"
 	"net/http/httptest"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -707,3 +708,30 @@ func TestOnlyANodeThatIsGoneCanBeRemoved(t *testing.T) {
 		t.Errorf("removing it again gave %d, expected 404", code)
 	}
 }
+
+func TestRemovingANodeRevokesItsCertificate(t *testing.T) {
+	a, ctl := setup(t, testModels(), gpuNode("n1", 8), gpuNode("n2", 8))
+	var revoked []string
+	a.SetRevoker(func(_ context.Context, id string, _ time.Time) error {
+		revoked = append(revoked, id)
+		return nil
+	})
+	ctl.Inventory().SetState("n1", inventory.Lost)
+	if code := call(t, a, "DELETE", "/api/v1/nodes/n1", "", nil); code != 200 {
+		t.Fatalf("a lost node gave %d, expected 200", code)
+	}
+	if !slices.Equal(revoked, []string{"n1"}) {
+		t.Errorf("revoked %v, expected n1", revoked)
+	}
+
+	// Not revoked, not removed: a node left listed is one the operator can
+	// see and try again on.
+	a.SetRevoker(func(context.Context, string, time.Time) error { return errors.New("store down") })
+	ctl.Inventory().SetState("n2", inventory.Lost)
+	if code := call(t, a, "DELETE", "/api/v1/nodes/n2", "", nil); code != 500 {
+		t.Errorf("a failed revocation gave %d, expected 500", code)
+	}
+	if _, ok := ctl.Inventory().Get("n2"); !ok {
+		t.Error("the node was removed though its certificate was not revoked")
+	}
+}