llmgw stop, start and restart act on the whole system

Issue #3 asked for commands to stop and start the system, and 05aa21c gave
only one machine's services. Now:

- llmgw stop sets serving.stopped in the cluster's settings, live: the
  reconciler wants every model at no replicas, freeing every GPU, and the
  gateway answers every POST under /v1/ with 503 system_stopped. The control
  planes, agents and mirror keep running, and no model's definition changes.
- llmgw start clears it; every model is placed again as it asks.
- llmgw restart has every agent restart over its stream (RestartAgent), then
  the other control planes over ssh, standbys before a leader, then this
  one; engines keep serving.
- llmgw status is the cluster's summary, saying when it is stopped.

One machine's services moved to llmgw service status|start|stop|restart.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-07 18:01:45 +02:00
1 parent 05aa21c4d4
commit 7245dbe1dd
29 files changed
+891 -254

No files matched your search

+14
View File
@@ -186,6 +186,7 @@ func newServerCmd() *cobra.Command {
rt.Start(ctx)
defer rt.Stop()
rt.SetStopped(cfg.Serving.Stopped)
// Which cluster this is, to federated peers: the id in the hop
// header that stops a request going round two clusters that
@@ -252,6 +253,8 @@ func newServerCmd() *cobra.Command {
}
defer ctl.Close()
agentSvc = svc
// Stopped by the operators (llmgw stop): nothing placed.
ctl.Reconciler().SetStopped(cfg.Serving.Stopped)
// Whether nodes keep llmgw's firewall, as the settings say now.
svc.SetFirewall(func() *llmgwv1.Firewall {
c := current.Load()
@@ -367,6 +370,9 @@ func newServerCmd() *cobra.Command {
if st := rt.Store(); st != nil {
// Removing a node revokes its certificate.
api.SetRevoker(st.RevokeNode)
if agentSvc != nil {
api.SetAgentRestarter(agentSvc.RestartAgents)
}
api.SetAudit(st)
// The cluster's settings, each change checked against this
// machine's own keys before it is kept.
@@ -676,6 +682,14 @@ func newServerCmd() *cobra.Command {
return false
}
current.Store(&next)
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "serving") }) {
rt.SetStopped(next.Serving.Stopped)
if admin != nil {
admin.Reconciler().SetStopped(next.Serving.Stopped)
}
log.Warn("the service is " + map[bool]string{true: "stopped", false: "started"}[next.Serving.Stopped] +
" by the cluster's settings")
}
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "firewall") }) {
if err := writeFirewallState(next); err != nil {
log.Warn("could not record the firewall's settings for this machine", "err", err)