llmgw stop, start and restart act on the whole system
Issue #3 asked for commands to stop and start the system, and 05aa21c gave
only one machine's services. Now:
- llmgw stop sets serving.stopped in the cluster's settings, live: the
reconciler wants every model at no replicas, freeing every GPU, and the
gateway answers every POST under /v1/ with 503 system_stopped. The control
planes, agents and mirror keep running, and no model's definition changes.
- llmgw start clears it; every model is placed again as it asks.
- llmgw restart has every agent restart over its stream (RestartAgent), then
the other control planes over ssh, standbys before a leader, then this
one; engines keep serving.
- llmgw status is the cluster's summary, saying when it is stopped.
One machine's services moved to llmgw service status|start|stop|restart.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
05aa21c4d4
commit
7245dbe1dd
29 files changed
+891
-254
No files matched your search
@@ -81,6 +81,10 @@ func (r *Reconciler) Want(model string) bool {
|
||||
// feature: a cluster with more models than GPUs can hold every model in its
|
||||
// catalog and only the wanted ones in its cards.
|
||||
func (r *Reconciler) desiredFor(m catalog.Model) int {
|
||||
// Stopped by the operators: nothing runs, whatever the catalog asks.
|
||||
if r.stopped.Load() {
|
||||
return 0
|
||||
}
|
||||
base := r.baseFor(m)
|
||||
if base == 0 {
|
||||
return 0
|
||||
|
||||
@@ -307,6 +307,9 @@ type Reconciler struct {
|
||||
startedAt time.Time
|
||||
lastNodeCount int
|
||||
steadySince time.Time
|
||||
// stopped is the operators having stopped the service: every model is
|
||||
// wanted at no replicas until they start it again.
|
||||
stopped atomic.Bool
|
||||
// stoodBy is whether the last pass found this process a standby: one
|
||||
// that then takes over settles again, as at start.
|
||||
stoodBy bool
|
||||
@@ -484,6 +487,17 @@ func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// SetStopped stops every model, or lets each run as its catalog entry
|
||||
// asks again, from the next pass.
|
||||
func (r *Reconciler) SetStopped(stopped bool) {
|
||||
if r.stopped.Swap(stopped) != stopped {
|
||||
r.Notify()
|
||||
}
|
||||
}
|
||||
|
||||
// Stopped is whether the operators have stopped the service.
|
||||
func (r *Reconciler) Stopped() bool { return r.stopped.Load() }
|
||||
|
||||
// resettle opens the settling window again from now, for a control plane
|
||||
// that has just become the leader.
|
||||
func (r *Reconciler) resettle() {
|
||||
|
||||
@@ -2133,3 +2133,27 @@ func TestAnAdoptedShardKeepsWhatItComputesOn(t *testing.T) {
|
||||
t.Fatalf("%+v", reps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoppingTheServiceTakesEveryModelDownAndStartingBringsItBack(t *testing.T) {
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
if len(h.r.ReadyReplicas()) != 1 {
|
||||
t.Fatal("nothing ready to stop")
|
||||
}
|
||||
h.r.SetStopped(true)
|
||||
h.r.Reconcile(t.Context())
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.State == Ready {
|
||||
t.Fatalf("stopped, a replica still serves: %+v", rep)
|
||||
}
|
||||
}
|
||||
h.r.SetStopped(false)
|
||||
for range 3 {
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
}
|
||||
if len(h.r.ReadyReplicas()) != 1 {
|
||||
t.Errorf("started again, %d ready", len(h.r.ReadyReplicas()))
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user