llmgw stop, start and restart act on the whole system

Issue #3 asked for commands to stop and start the system, and 05aa21c gave
only one machine's services. Now:

- llmgw stop sets serving.stopped in the cluster's settings, live: the
  reconciler wants every model at no replicas, freeing every GPU, and the
  gateway answers every POST under /v1/ with 503 system_stopped. The control
  planes, agents and mirror keep running, and no model's definition changes.
- llmgw start clears it; every model is placed again as it asks.
- llmgw restart has every agent restart over its stream (RestartAgent), then
  the other control planes over ssh, standbys before a leader, then this
  one; engines keep serving.
- llmgw status is the cluster's summary, saying when it is stopped.

One machine's services moved to llmgw service status|start|stop|restart.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-07 18:01:45 +02:00
1 parent 05aa21c4d4
commit 7245dbe1dd
29 files changed
+891 -254

No files matched your search

+2
View File
@@ -237,6 +237,8 @@ func (w *settingsWatch) check(ctx context.Context) {
// again: what it reads at every use, or not at all.
var liveSettings = []string{
"guard",
// Stopping and starting the service (llmgw stop, llmgw start).
"serving",
// Sent to the nodes and this machine's firewall unit as they change.
"firewall", "firewall_networks",
// Read by server init, node add and backups, not by the server.
+4 -2
View File
@@ -63,8 +63,10 @@ func Root() *cobra.Command {
newWipeCmd(),
newDocsCmd(),
)
// status, start, stop and restart: this machine's llmgw services.
root.AddCommand(newServiceCmds()...)
// status, start, stop and restart: the whole system; service: this
// machine's services.
root.AddCommand(newSystemCmds()...)
root.AddCommand(newServiceCmd())
return root
}
+14
View File
@@ -186,6 +186,7 @@ func newServerCmd() *cobra.Command {
rt.Start(ctx)
defer rt.Stop()
rt.SetStopped(cfg.Serving.Stopped)
// Which cluster this is, to federated peers: the id in the hop
// header that stops a request going round two clusters that
@@ -252,6 +253,8 @@ func newServerCmd() *cobra.Command {
}
defer ctl.Close()
agentSvc = svc
// Stopped by the operators (llmgw stop): nothing placed.
ctl.Reconciler().SetStopped(cfg.Serving.Stopped)
// Whether nodes keep llmgw's firewall, as the settings say now.
svc.SetFirewall(func() *llmgwv1.Firewall {
c := current.Load()
@@ -367,6 +370,9 @@ func newServerCmd() *cobra.Command {
if st := rt.Store(); st != nil {
// Removing a node revokes its certificate.
api.SetRevoker(st.RevokeNode)
if agentSvc != nil {
api.SetAgentRestarter(agentSvc.RestartAgents)
}
api.SetAudit(st)
// The cluster's settings, each change checked against this
// machine's own keys before it is kept.
@@ -676,6 +682,14 @@ func newServerCmd() *cobra.Command {
return false
}
current.Store(&next)
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "serving") }) {
rt.SetStopped(next.Serving.Stopped)
if admin != nil {
admin.Reconciler().SetStopped(next.Serving.Stopped)
}
log.Warn("the service is " + map[bool]string{true: "stopped", false: "started"}[next.Serving.Stopped] +
" by the cluster's settings")
}
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "firewall") }) {
if err := writeFirewallState(next); err != nil {
log.Warn("could not record the firewall's settings for this machine", "err", err)
+16 -3
View File
@@ -78,13 +78,26 @@ func hasUnit(list []string, s string) bool {
return false
}
func newServiceCmds() []*cobra.Command {
// newServiceCmd is llmgw service: this machine's llmgw services, one by one.
// The whole system is llmgw status, start, stop and restart.
func newServiceCmd() *cobra.Command {
c := &cobra.Command{
Use: "service",
Short: "This machine's llmgw services: status, start, stop, restart",
Long: "The control plane (server), node agent (agent) and model mirror (mirror) on\n" +
"this machine alone, by systemd. The whole system is 'llmgw status', 'llmgw\n" +
"stop', 'llmgw start' and 'llmgw restart', run on a control plane.",
}
c.AddCommand(serviceVerbs()...)
return c
}
func serviceVerbs() []*cobra.Command {
status := &cobra.Command{
Use: "status [server|agent|mirror]...",
Short: "Whether llmgw's services on this machine run",
Long: "Each llmgw service this machine has — the control plane (server), the\n" +
"node agent (agent), the model mirror (mirror) — and whether it runs.\n" +
"The cluster as a whole is 'llmgw ctl status'.",
"node agent (agent), the model mirror (mirror) — and whether it runs.",
RunE: func(cmd *cobra.Command, args []string) error {
units, err := localUnits(cmd, args)
if err != nil {
+51 -1
View File
@@ -3,11 +3,18 @@ package cli
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
func TestTheServicesHereAreFoundAndAskedAbout(t *testing.T) {
@@ -42,7 +49,7 @@ func TestTheServicesHereAreFoundAndAskedAbout(t *testing.T) {
return
}
var out bytes.Buffer
c := newServiceCmds()[0]
c := serviceVerbs()[0]
c.SetOut(&out)
c.SetContext(context.Background())
if err := c.RunE(c, nil); err != nil {
@@ -52,3 +59,46 @@ func TestTheServicesHereAreFoundAndAskedAbout(t *testing.T) {
t.Errorf("%s", out.String())
}
}
func TestStopSetsTheClustersServingStopped(t *testing.T) {
var put adminapi.SettingsRequest
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.URL.Path == "/api/v1/summary":
_ = json.NewEncoder(w).Encode(adminapi.Summary{})
case r.Method == "GET" && r.URL.Path == "/api/v1/settings":
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 3, Body: "router:\n retries: 2\n"})
case r.Method == "PUT" && r.URL.Path == "/api/v1/settings":
_ = json.NewDecoder(r.Body).Decode(&put)
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 4, Body: put.Body})
default:
http.NotFound(w, r)
}
}))
defer srv.Close()
dir := t.TempDir()
cfg := filepath.Join(dir, "server.yaml")
_ = os.WriteFile(filepath.Join(dir, "token"), []byte("t"), 0o600)
_ = os.WriteFile(cfg, []byte("listen: \":8080\"\nagent_insecure: true\nadmin_listen: "+
strings.TrimPrefix(srv.URL, "http://")+"\nadmin_remote: true\nadmin_token_file: "+filepath.Join(dir, "token")+"\n"), 0o600)
var stop *cobra.Command
for _, c := range newSystemCmds() {
if c.Name() == "stop" {
stop = c
}
}
var out bytes.Buffer
stop.SetOut(&out)
stop.SetArgs([]string{"--config", cfg, "--note", "maintenance"})
if err := stop.ExecuteContext(context.Background()); err != nil {
t.Fatalf("%v: %s", err, out.String())
}
if put.Base != 3 || !strings.Contains(put.Body, "stopped: true") || !strings.Contains(put.Body, "retries: 2") ||
put.Note != "maintenance" {
t.Errorf("sent %+v", put)
}
if !strings.Contains(out.String(), "stopping") {
t.Errorf("%s", out.String())
}
}
+3
View File
@@ -153,6 +153,9 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) {
fmt.Fprintf(out, "STORE %s\n", strings.Join(parts, ", "))
}
if s.Stopped {
fmt.Fprintln(out, "SERVICE STOPPED: no model runs and requests are answered 503 (llmgw start)")
}
if s.GuardOff {
fmt.Fprintln(out, "GUARD off: no request is checked, whatever the policies say (llmgw ctl guard on)")
}
+216
View File
@@ -0,0 +1,216 @@
package cli
import (
"errors"
"fmt"
"os"
"os/exec"
"strings"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
)
// systemClient is the admin API of the control plane on this machine, as its
// server.yaml says, with its token.
func systemClient(configPath string) func() *ctlClient {
return func() *ctlClient {
addr := "127.0.0.1:9090"
var tokens []string
if cfg, ok := loadServerConfig(configPath); ok {
if cfg.AdminListen != "" {
addr = cfg.AdminListen
if strings.HasPrefix(addr, ":") {
addr = "127.0.0.1" + addr
}
}
tokens = append(tokens, adminTokenPath(configPath, cfg))
}
return newCtlClient(addr, tokens...)
}
}
// newSystemCmds are llmgw status, stop, start and restart: the whole system,
// from a control plane.
func newSystemCmds() []*cobra.Command {
var configPath string
client := systemClient("/etc/llmgw/server.yaml")
bind := func(c *cobra.Command) *cobra.Command {
c.Flags().StringVar(&configPath, "config", "/etc/llmgw/server.yaml", "this control plane's configuration")
c.PreRun = func(*cobra.Command, []string) { client = systemClient(configPath) }
return c
}
call := func() *ctlClient { return client() }
status := bind(&cobra.Command{
Use: "status",
Short: "The whole system: whether it serves, its control planes, nodes, models and store",
Long: "Run on a control plane. Says whether the service is stopped (llmgw stop),\n" +
"and everything 'llmgw ctl status' does. One machine's services are\n" +
"'llmgw service status'.",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
var s adminapi.Summary
if err := call().get(cmd.Context(), "/api/v1/summary", &s); err != nil {
return notAControlPlane(err)
}
printSummary(cmd, s)
return nil
},
})
switchService := func(stop bool) *cobra.Command {
use, short, long := "start", "Start the whole system serving again, after llmgw stop",
"Every model comes back as its catalog entry asks: placed, loaded and\n"+
"serving again, on every control plane within seconds. Run on a control\n"+
"plane. The services themselves are never stopped by llmgw stop; a\n"+
"machine's own are started with 'llmgw service start'."
if stop {
use, short, long = "stop", "Stop the whole system serving: every model taken down, requests answered 503",
"Every model's replicas are taken down, freeing every GPU, and every\n"+
"request for a model is answered 503 (system_stopped) until 'llmgw start'.\n"+
"The control planes, agents and mirror keep running, so the console, the\n"+
"admin pages and llmgw start still work, and no model's definition is\n"+
"changed. In force on every control plane within seconds; kept in the\n"+
"cluster's settings (serving.stopped), so a restart does not undo it."
}
var note string
var wait bool
c := bind(&cobra.Command{
Use: use,
Short: short,
Long: long,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
if note == "" {
note = "llmgw " + use
}
var s adminapi.Summary
if err := call().get(cmd.Context(), "/api/v1/summary", &s); err != nil {
return notAControlPlane(err)
}
if s.Stopped == stop {
fmt.Fprintf(cmd.OutOrStdout(), "the system is already %s\n", map[bool]string{true: "stopped", false: "serving"}[stop])
return nil
}
if err := changeSettings(cmd, call, note, func(doc *settings.Doc) error {
return doc.SetYAML([]string{"serving", "stopped"}, fmt.Sprint(stop))
}); err != nil {
return err
}
out := cmd.OutOrStdout()
if !wait {
if stop {
fmt.Fprintln(out, "stopping: every model's replicas are being taken down; llmgw status follows it")
} else {
fmt.Fprintln(out, "starting: every model is placed again as it asks; llmgw status follows it")
}
return nil
}
return waitForService(cmd, call, stop)
},
})
c.Flags().StringVar(&note, "note", "", "why, kept with the settings' version and in the audit")
c.Flags().BoolVar(&wait, "wait", false, "wait until every replica is down (stop) or the models have theirs (start)")
return c
}
var sshUser string
restart := bind(&cobra.Command{
Use: "restart",
Short: "Restart every control plane and node agent; the engines keep serving",
Long: "Every node's agent starts again, then the other control planes over ssh\n" +
"as --ssh-user, then this one — the leader last, so it hands over to a\n" +
"standby already restarted. The engines keep serving throughout and are\n" +
"taken over again; nothing is placed anew. Run on a control plane, with\n" +
"sudo. One machine's services are 'llmgw service restart'.",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
if os.Geteuid() != 0 {
return errors.New("restarting this control plane needs root: run it with sudo")
}
ctx, out := cmd.Context(), cmd.OutOrStdout()
var s adminapi.Summary
if err := call().get(ctx, "/api/v1/summary", &s); err != nil {
return notAControlPlane(err)
}
var agents adminapi.AgentRestartView
if err := call().post(ctx, "/api/v1/agents/restart", []byte("{}"), &agents); err != nil {
return fmt.Errorf("restarting the agents: %w", err)
}
fmt.Fprintf(out, "agents %d told to restart: %s\n", len(agents.Nodes), strings.Join(agents.Nodes, ", "))
// The other control planes, standbys before a leader.
var others []adminapi.ControlPlane
for _, cp := range s.ControlPlanes.Members {
if !cp.This {
others = append(others, cp)
}
}
for pass := 0; pass < 2; pass++ {
for _, cp := range others {
leading := cp.State == "leading"
if leading != (pass == 1) {
continue
}
r := &onboard.SSH{Addr: cp.Address, User: sshUser}
if res, err := r.Run(ctx, "sudo systemctl restart llmgw-server", nil); err != nil {
return fmt.Errorf("%s (%s) could not be restarted over ssh as %q: %v %s; "+
"run `sudo llmgw service restart server` there", cp.Name, cp.Address, sshUser, err, strings.TrimSpace(string(res)))
}
fmt.Fprintf(out, "%-10s restarted (%s)\n", cp.Name, cp.State)
time.Sleep(5 * time.Second)
}
}
if _, err := os.Stat(unitDir + "/llmgw-server.service"); err == nil {
fmt.Fprintln(out, "this restarting")
if b, err := exec.CommandContext(ctx, "systemctl", "restart", "llmgw-server").CombinedOutput(); err != nil { //nolint:gosec // fixed command
return fmt.Errorf("restarting this control plane: %v: %s", err, strings.TrimSpace(string(b)))
}
fmt.Fprintln(out, "this restarted")
}
fmt.Fprintln(out, "\nllmgw status shows the nodes coming back; their replicas are taken over as they are.")
return nil
},
})
restart.Flags().StringVar(&sshUser, "ssh-user", os.Getenv("SUDO_USER"), "who to log in to the other control planes as")
return []*cobra.Command{status, switchService(true), switchService(false), restart}
}
// waitForService waits until no replica runs (stopped) or every model has
// the replicas it asks for (started).
func waitForService(cmd *cobra.Command, call func() *ctlClient, stop bool) error {
deadline := time.Now().Add(15 * time.Minute)
for time.Now().Before(deadline) {
var s adminapi.Summary
if err := call().get(cmd.Context(), "/api/v1/summary", &s); err == nil {
if stop && s.Replicas.Ready+s.Replicas.Starting+s.Replicas.Draining == 0 {
fmt.Fprintln(cmd.OutOrStdout(), "stopped: no replica runs")
return nil
}
if !stop && s.Models.Satisfied >= s.Models.Total {
fmt.Fprintf(cmd.OutOrStdout(), "serving: %d of %d models have the replicas they ask for\n",
s.Models.Satisfied, s.Models.Total)
return nil
}
}
select {
case <-cmd.Context().Done():
return cmd.Context().Err()
case <-time.After(5 * time.Second):
}
}
return errors.New("not there within fifteen minutes; llmgw status says where it stands")
}
// notAControlPlane is a failure to reach the admin API, said as what it
// most often means.
func notAControlPlane(err error) error {
return fmt.Errorf("%w\n(the whole system is stopped and started from a control plane; this machine's own "+
"services are 'llmgw service status|start|stop|restart')", err)
}