llmgw stop, start and restart act on the whole system
Issue #3 asked for commands to stop and start the system, and 05aa21c gave
only one machine's services. Now:
- llmgw stop sets serving.stopped in the cluster's settings, live: the
reconciler wants every model at no replicas, freeing every GPU, and the
gateway answers every POST under /v1/ with 503 system_stopped. The control
planes, agents and mirror keep running, and no model's definition changes.
- llmgw start clears it; every model is placed again as it asks.
- llmgw restart has every agent restart over its stream (RestartAgent), then
the other control planes over ssh, standbys before a leader, then this
one; engines keep serving.
- llmgw status is the cluster's summary, saying when it is stopped.
One machine's services moved to llmgw service status|start|stop|restart.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
05aa21c4d4
commit
7245dbe1dd
29 files changed
+891
-254
No files matched your search
@@ -310,7 +310,7 @@ func (r *Router) Handler() http.Handler {
|
||||
}
|
||||
|
||||
if r.auth == nil {
|
||||
return r.refuseLoops(mux)
|
||||
return r.refuseWhileStopped(r.refuseLoops(mux))
|
||||
}
|
||||
exempt := []string{"/health", "/ready", "/leader", "/", "/about", "/_static/", "/_lang/", "/claim/",
|
||||
"/signup", "/login", "/login/", "/logout", "/profile", "/profile/", "/scim/",
|
||||
@@ -324,7 +324,22 @@ func (r *Router) Handler() http.Handler {
|
||||
// Wrapping the whole mux rather than checking inside each handler: adding a
|
||||
// route must not be able to add an unauthenticated one by omission. Health
|
||||
// stays open because a load balancer has no key.
|
||||
return r.refuseLoops(auth.Middleware(r.auth, r.log, exempt...)(mux))
|
||||
return r.refuseWhileStopped(r.refuseLoops(auth.Middleware(r.auth, r.log, exempt...)(mux)))
|
||||
}
|
||||
|
||||
// refuseWhileStopped answers every request for a model with 503 while the
|
||||
// operators have stopped the service (llmgw stop): the models are being taken
|
||||
// down, and a client is better told so plainly than left to time out.
|
||||
func (r *Router) refuseWhileStopped(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
if r.stopped.Load() && req.Method == http.MethodPost && strings.HasPrefix(req.URL.Path, "/v1/") {
|
||||
w.Header().Set("Retry-After", "300")
|
||||
writeErrorCode(w, http.StatusServiceUnavailable, "server_error", "system_stopped",
|
||||
"this gateway's service has been stopped by its operators; it answers again once they start it")
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, req)
|
||||
})
|
||||
}
|
||||
|
||||
// parsedRequest is what the router needs from a request. Everything else in the
|
||||
|
||||
@@ -115,7 +115,9 @@ type Router struct {
|
||||
oidc config.OIDC
|
||||
// mail sends what the portal mails: a password reset link.
|
||||
mail mail.Config
|
||||
scim config.SCIM
|
||||
// stopped is the operators having stopped the service (llmgw stop).
|
||||
stopped atomic.Bool
|
||||
scim config.SCIM
|
||||
|
||||
// threads holds conversations and memory notes. It is store when auth is on,
|
||||
// its own file when only the threads block is configured, and nil when
|
||||
@@ -192,6 +194,9 @@ func (r *Router) SetGuard(g guard.Guard) { r.guard = g }
|
||||
// SetGuardLibrary is the policies the guard was built from, for the pages.
|
||||
func (r *Router) SetGuardLibrary(c func() guard.Config) { r.guardLibrary = c }
|
||||
|
||||
// SetStopped refuses, or again serves, every request for a model.
|
||||
func (r *Router) SetStopped(stopped bool) { r.stopped.Store(stopped) }
|
||||
|
||||
// SetRegistry gives the mirror page somewhere to look models up.
|
||||
func (r *Router) SetRegistry(reg web.Registry) { r.registry = reg }
|
||||
|
||||
|
||||
@@ -748,3 +748,24 @@ func TestModelsEndpointListsOnlyWhatTheKeyMayUse(t *testing.T) {
|
||||
t.Errorf("listed %v; the key may use real and its alias only", ids)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStoppedServiceAnswers503AndStillListsModels(t *testing.T) {
|
||||
e := newFakeEngine(t, "m")
|
||||
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}))
|
||||
h := rt.Handler()
|
||||
rt.SetStopped(true)
|
||||
rec := post(t, h, map[string]any{"model": "m", "messages": []map[string]string{{"role": "user", "content": "hi"}}}, nil)
|
||||
if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "system_stopped") ||
|
||||
rec.Header().Get("Retry-After") == "" {
|
||||
t.Fatalf("stopped: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
list := httptest.NewRecorder()
|
||||
h.ServeHTTP(list, httptest.NewRequest(http.MethodGet, "/v1/models", nil))
|
||||
if list.Code == http.StatusServiceUnavailable {
|
||||
t.Error("listing models was refused too")
|
||||
}
|
||||
rt.SetStopped(false)
|
||||
if rec := post(t, h, map[string]any{"model": "m", "messages": []map[string]string{{"role": "user", "content": "hi"}}}, nil); rec.Code == http.StatusServiceUnavailable {
|
||||
t.Errorf("started again, still refused: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user