Restart replicas without taking them out of service first

There was no way to restart a running replica. A running engine picks up an
engine key, an edited profile or a new engine version only when it starts,
so after v0.10.3 the nine replicas already running stayed without a key —
reachable from any node — with nothing to do about it short of draining
nodes, which on a small CPU fleet has nowhere to move a model to.

`llmgw ctl restart REPLICA... | --model M | --all` queues replicas to be
replaced. A replica being replaced keeps serving but stops counting toward
its model, so the reconciler places a replacement beside it; once that is
ready the old one is drained and stopped as a node drain would. Replicas of
one model go one at a time, so a model never needs the memory of every
replica twice. Where no second copy fits anywhere, the old replica is
stopped after the drain grace period and the new one starts in its place,
and the log says so, since that is the one case with an outage.

`ctl replicas` shows "(restart queued)" and "(restart replacing)" while it
happens.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

internal/control/adminapi/api.go

@@ -77,6 +77,7 @@ func (a *API) Register(mux *http.ServeMux) {
 	mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
 	mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
 	mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
+	mux.HandleFunc("POST /api/v1/restart", a.restart)
 }
 
 // --- responses ---
@@ -243,8 +244,11 @@ type ReplicaView struct {
 	// and anyone else holding the admin listener who needs to reach an engine
 	// directly. The client-facing pages never render it.
 	EngineKey string `json:"engine_key,omitempty"`
-	Model     string `json:"model"`
-	Profile   string `json:"profile"`
+	// Restart is "queued" or "replacing" while an operator's restart of this
+	// replica is under way.
+	Restart string `json:"restart,omitempty"`
+	Model   string `json:"model"`
+	Profile string `json:"profile"`
 	// Stale is true when the profile has been edited since this replica was
 	// placed, so it is running a definition the catalog no longer holds.
 	//
@@ -476,6 +480,7 @@ func (a *API) Replicas(model string) []ReplicaView {
 			State: rep.State.String(), Pool: rep.Pool, Fabric: rep.Fabric,
 			Endpoint: rep.Endpoint, Err: rep.Err,
 			EngineKey: a.ctl.EngineKey(rep.ID),
+			Restart:   a.ctl.Reconciler().RestartState(rep.ID),
 			Stale:     a.profileChanged(rep.Model, rep.Profile, rep.ProfileFP),
 			AgeSec:    int64(time.Since(rep.CreatedAt).Seconds()),
 			IdleSec:   -1,
@@ -1044,3 +1049,39 @@ func (a *API) Topology(id string) (TopologyView, bool) {
 func (a *API) NVLink() nvlink.Report {
 	return nvlink.Analyze(a.ctl.Inventory().Nodes())
 }
+
+// RestartRequest names what to restart: replicas by id, every replica of some
+// models, or — All — every ready replica in the cluster.
+type RestartRequest struct {
+	Replicas []string `json:"replicas,omitempty"`
+	Models   []string `json:"models,omitempty"`
+	All      bool     `json:"all,omitempty"`
+}
+
+// RestartResult lists the replicas queued.
+type RestartResult struct {
+	Queued []string `json:"queued"`
+}
+
+func (a *API) restart(w http.ResponseWriter, r *http.Request) {
+	var req RestartRequest
+	if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, "the restart request is not valid JSON: "+err.Error())
+		return
+	}
+	ids := slices.Clone(req.Replicas)
+	for _, rep := range a.ctl.Reconciler().ReadyReplicas() {
+		if req.All || slices.Contains(req.Models, rep.Model) {
+			ids = append(ids, rep.ID)
+		}
+	}
+	if len(ids) == 0 {
+		writeError(w, http.StatusBadRequest, "nothing ready matches; name replicas, models, or all")
+		return
+	}
+	if err := a.ctl.Reconciler().Restart(ids...); err != nil {
+		writeError(w, http.StatusNotFound, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusAccepted, RestartResult{Queued: ids})
+}