llmgw stop, start and restart act on the whole system

Issue #3 asked for commands to stop and start the system, and 05aa21c gave
only one machine's services. Now:

- llmgw stop sets serving.stopped in the cluster's settings, live: the
  reconciler wants every model at no replicas, freeing every GPU, and the
  gateway answers every POST under /v1/ with 503 system_stopped. The control
  planes, agents and mirror keep running, and no model's definition changes.
- llmgw start clears it; every model is placed again as it asks.
- llmgw restart has every agent restart over its stream (RestartAgent), then
  the other control planes over ssh, standbys before a leader, then this
  one; engines keep serving.
- llmgw status is the cluster's summary, saying when it is stopped.

One machine's services moved to llmgw service status|start|stop|restart.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-07 18:01:45 +02:00
1 parent 05aa21c4d4
commit 7245dbe1dd
29 files changed
+891 -254

No files matched your search

+3
View File
@@ -70,6 +70,8 @@ type API struct {
// guard is the guardrail policies' library and attachments.
guard *GuardAdmin
settings *Settings
// restartAgents tells every connected agent to start again.
restartAgents func(reason string) []string
// revoke records a removed node, so its certificate is refused.
revoke func(ctx context.Context, nodeID string, at time.Time) error
@@ -114,6 +116,7 @@ func (a *API) Register(mux Mux) {
a.registerQuotas(mux)
a.registerGuard(mux)
a.registerSettings(mux)
mux.HandleFunc("POST /api/v1/agents/restart", a.restartAgentsHandler)
mux.HandleFunc("GET /api/v1/audit", a.auditEvents)
mux.HandleFunc("GET /api/v1/usage/export", a.usageExport)
mux.HandleFunc("POST /api/v1/notify/test", a.notifyTest)
+3
View File
@@ -263,6 +263,9 @@ var Routes = []Route{
{Method: "DELETE", Path: "/api/v1/guard/attachments", Summary: "Detach a guardrail policy",
Request: GuardAttachmentRequest{}, Response: guard.Attachment{}},
{Method: "POST", Path: "/api/v1/agents/restart", Summary: "Have every node's agent start again; their engines keep serving",
Response: AgentRestartView{}},
// The cluster's settings: everything in server.yaml that is not one
// machine's own, the same on every control plane.
{Method: "GET", Path: "/api/v1/settings", Summary: "The cluster's settings in force, or one earlier version",
+4 -1
View File
@@ -44,6 +44,9 @@ type Summary struct {
ControlPlanes ControlPlanes `json:"control_planes,omitzero"`
// Judges are the guardrails' judge models with something wrong.
Judges []JudgeView `json:"judges,omitempty"`
// Stopped says the operators have stopped the service (llmgw stop): no
// model runs, and requests for one are answered 503.
Stopped bool `json:"stopped,omitempty"`
// GuardOff says the guardrails are turned off (guard.enabled: false):
// no request is checked, whatever the policies say.
GuardOff bool `json:"guard_off,omitempty"`
@@ -153,7 +156,7 @@ const idleThreshold = 10 * time.Minute
// Summary builds the fleet summary.
func (a *API) Summary() Summary {
s := Summary{Version: version.Version}
s := Summary{Version: version.Version, Stopped: a.ctl.Reconciler().Stopped()}
if a.guard != nil && a.guard.Library != nil && !a.guard.Library().On() {
s.GuardOff = true
}
+22
View File
@@ -0,0 +1,22 @@
package adminapi
import (
"net/http"
)
// AgentRestartView names the nodes whose agents were told to restart.
type AgentRestartView struct {
Nodes []string `json:"nodes"`
}
// SetAgentRestarter enables POST /api/v1/agents/restart.
func (a *API) SetAgentRestarter(f func(reason string) []string) { a.restartAgents = f }
func (a *API) restartAgentsHandler(w http.ResponseWriter, r *http.Request) {
if a.restartAgents == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no agents to restart")
return
}
nodes := a.restartAgents("llmgw restart, by " + actorOf(r))
writeJSON(w, http.StatusAccepted, AgentRestartView{Nodes: nodes})
}
+19
View File
@@ -1,6 +1,8 @@
package agentsvc
import (
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
)
@@ -40,3 +42,20 @@ func (s *Service) BroadcastFirewall() {
}
}
}
// RestartAgents has every connected agent start again, one at a time a
// moment apart, and names those it told. Their engines keep serving.
func (s *Service) RestartAgents(reason string) []string {
msg := &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_RestartAgent{
RestartAgent: &llmgwv1.RestartAgent{Reason: reason}}}
var told []string
for _, node := range s.ConnectedNodes() {
if err := s.send(node, msg); err != nil {
s.log.Warn("could not tell a node to restart its agent", "node", node, "err", err)
continue
}
told = append(told, node)
time.Sleep(time.Second)
}
return told
}
+4
View File
@@ -81,6 +81,10 @@ func (r *Reconciler) Want(model string) bool {
// feature: a cluster with more models than GPUs can hold every model in its
// catalog and only the wanted ones in its cards.
func (r *Reconciler) desiredFor(m catalog.Model) int {
// Stopped by the operators: nothing runs, whatever the catalog asks.
if r.stopped.Load() {
return 0
}
base := r.baseFor(m)
if base == 0 {
return 0
+14
View File
@@ -307,6 +307,9 @@ type Reconciler struct {
startedAt time.Time
lastNodeCount int
steadySince time.Time
// stopped is the operators having stopped the service: every model is
// wanted at no replicas until they start it again.
stopped atomic.Bool
// stoodBy is whether the last pass found this process a standby: one
// that then takes over settles again, as at start.
stoodBy bool
@@ -484,6 +487,17 @@ func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) {
}
}
// SetStopped stops every model, or lets each run as its catalog entry
// asks again, from the next pass.
func (r *Reconciler) SetStopped(stopped bool) {
if r.stopped.Swap(stopped) != stopped {
r.Notify()
}
}
// Stopped is whether the operators have stopped the service.
func (r *Reconciler) Stopped() bool { return r.stopped.Load() }
// resettle opens the settling window again from now, for a control plane
// that has just become the leader.
func (r *Reconciler) resettle() {
@@ -2133,3 +2133,27 @@ func TestAnAdoptedShardKeepsWhatItComputesOn(t *testing.T) {
t.Fatalf("%+v", reps)
}
}
func TestStoppingTheServiceTakesEveryModelDownAndStartingBringsItBack(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Reconcile(t.Context())
h.markReady()
if len(h.r.ReadyReplicas()) != 1 {
t.Fatal("nothing ready to stop")
}
h.r.SetStopped(true)
h.r.Reconcile(t.Context())
for _, rep := range h.r.Replicas() {
if rep.State == Ready {
t.Fatalf("stopped, a replica still serves: %+v", rep)
}
}
h.r.SetStopped(false)
for range 3 {
h.r.Reconcile(t.Context())
h.markReady()
}
if len(h.r.ReadyReplicas()) != 1 {
t.Errorf("started again, %d ready", len(h.r.ReadyReplicas()))
}
}