llmgw stop, start and restart act on the whole system

Issue #3 asked for commands to stop and start the system, and 05aa21c gave
only one machine's services. Now:

- llmgw stop sets serving.stopped in the cluster's settings, live: the
  reconciler wants every model at no replicas, freeing every GPU, and the
  gateway answers every POST under /v1/ with 503 system_stopped. The control
  planes, agents and mirror keep running, and no model's definition changes.
- llmgw start clears it; every model is placed again as it asks.
- llmgw restart has every agent restart over its stream (RestartAgent), then
  the other control planes over ssh, standbys before a leader, then this
  one; engines keep serving.
- llmgw status is the cluster's summary, saying when it is stopped.

One machine's services moved to llmgw service status|start|stop|restart.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-07 18:01:45 +02:00
1 parent 05aa21c4d4
commit 7245dbe1dd
29 files changed
+891 -254

No files matched your search

+17 -2
View File
@@ -310,7 +310,7 @@ func (r *Router) Handler() http.Handler {
}
if r.auth == nil {
return r.refuseLoops(mux)
return r.refuseWhileStopped(r.refuseLoops(mux))
}
exempt := []string{"/health", "/ready", "/leader", "/", "/about", "/_static/", "/_lang/", "/claim/",
"/signup", "/login", "/login/", "/logout", "/profile", "/profile/", "/scim/",
@@ -324,7 +324,22 @@ func (r *Router) Handler() http.Handler {
// Wrapping the whole mux rather than checking inside each handler: adding a
// route must not be able to add an unauthenticated one by omission. Health
// stays open because a load balancer has no key.
return r.refuseLoops(auth.Middleware(r.auth, r.log, exempt...)(mux))
return r.refuseWhileStopped(r.refuseLoops(auth.Middleware(r.auth, r.log, exempt...)(mux)))
}
// refuseWhileStopped answers every request for a model with 503 while the
// operators have stopped the service (llmgw stop): the models are being taken
// down, and a client is better told so plainly than left to time out.
func (r *Router) refuseWhileStopped(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if r.stopped.Load() && req.Method == http.MethodPost && strings.HasPrefix(req.URL.Path, "/v1/") {
w.Header().Set("Retry-After", "300")
writeErrorCode(w, http.StatusServiceUnavailable, "server_error", "system_stopped",
"this gateway's service has been stopped by its operators; it answers again once they start it")
return
}
next.ServeHTTP(w, req)
})
}
// parsedRequest is what the router needs from a request. Everything else in the
+6 -1
View File
@@ -115,7 +115,9 @@ type Router struct {
oidc config.OIDC
// mail sends what the portal mails: a password reset link.
mail mail.Config
scim config.SCIM
// stopped is the operators having stopped the service (llmgw stop).
stopped atomic.Bool
scim config.SCIM
// threads holds conversations and memory notes. It is store when auth is on,
// its own file when only the threads block is configured, and nil when
@@ -192,6 +194,9 @@ func (r *Router) SetGuard(g guard.Guard) { r.guard = g }
// SetGuardLibrary is the policies the guard was built from, for the pages.
func (r *Router) SetGuardLibrary(c func() guard.Config) { r.guardLibrary = c }
// SetStopped refuses, or again serves, every request for a model.
func (r *Router) SetStopped(stopped bool) { r.stopped.Store(stopped) }
// SetRegistry gives the mirror page somewhere to look models up.
func (r *Router) SetRegistry(reg web.Registry) { r.registry = reg }
+21
View File
@@ -748,3 +748,24 @@ func TestModelsEndpointListsOnlyWhatTheKeyMayUse(t *testing.T) {
t.Errorf("listed %v; the key may use real and its alias only", ids)
}
}
func TestAStoppedServiceAnswers503AndStillListsModels(t *testing.T) {
e := newFakeEngine(t, "m")
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}))
h := rt.Handler()
rt.SetStopped(true)
rec := post(t, h, map[string]any{"model": "m", "messages": []map[string]string{{"role": "user", "content": "hi"}}}, nil)
if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "system_stopped") ||
rec.Header().Get("Retry-After") == "" {
t.Fatalf("stopped: %d %s", rec.Code, rec.Body.String())
}
list := httptest.NewRecorder()
h.ServeHTTP(list, httptest.NewRequest(http.MethodGet, "/v1/models", nil))
if list.Code == http.StatusServiceUnavailable {
t.Error("listing models was refused too")
}
rt.SetStopped(false)
if rec := post(t, h, map[string]any{"model": "m", "messages": []map[string]string{{"role": "user", "content": "hi"}}}, nil); rec.Code == http.StatusServiceUnavailable {
t.Errorf("started again, still refused: %s", rec.Body.String())
}
}