llmgw status, start, stop and restart for this machine's services

There was no command to stop or start llmgw on a machine (issue #3). These
act on the llmgw units the machine has - mirror, server, agent - or those
named, in dependency order, saying what each did. stop leaves a node's
engines serving unless --engines.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

CHANGELOG.md

@@ -28,6 +28,10 @@ What was left of the field report from v0.10.6.
 
 ### Added
 
+- **`llmgw status`, `start`, `stop` and `restart`** for this machine's llmgw
+  services — server, agent, mirror, or those named — in the order they
+  depend on each other. Stopping the agent leaves the engines serving unless
+  `--engines` is given; stopping a leader hands over to a standby.
 - **A replica says what it computes on**: `ctl replicas` (ON), the console
   and the admin API show `CUDA` with the GPUs its processes hold memory on
   and how much, as nvidia-smi sees them, or `CPU` — for every engine, kept by

docs/install.md

@@ -2294,6 +2294,21 @@ A cluster set up online becomes offline the same way: `bundle import` on the
 mirror and control plane, then the `offline:` block in `server.yaml` and a
 restart.
 
+## Stopping and starting a machine
+
+```
+sudo llmgw status              # each llmgw service here, and whether it runs
+sudo llmgw stop                # agent, control plane, mirror — whichever this machine has
+sudo llmgw start
+sudo llmgw restart agent       # or name one: server, agent, mirror
+```
+
+Stopping a node's agent leaves its engines serving — the agent takes them
+over again when it starts — unless `llmgw stop --engines` stops them too, and
+the control plane then places their replicas elsewhere. Stopping the leading
+control plane hands over to a standby within seconds. The whole cluster's
+state is `llmgw ctl status`.
+
 ## Removing llmgw from a machine
 
 ```

internal/cli/root.go

@@ -63,6 +63,8 @@ func Root() *cobra.Command {
 		newWipeCmd(),
 		newDocsCmd(),
 	)
+	// status, start, stop and restart: this machine's llmgw services.
+	root.AddCommand(newServiceCmds()...)
 	return root
 }
 

internal/cli/service.go

@@ -0,0 +1,174 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+
+	"github.com/spf13/cobra"
+)
+
+// llmgwServices are the units llmgw runs as on a machine, in the order they
+// start: the store's clients after the store, the server before the agent
+// that dials it.
+var llmgwServices = []string{"llmgw-mirror", "llmgw-server", "llmgw-agent"}
+
+// unitDir is where llmgw's units are installed.
+var unitDir = "/etc/systemd/system"
+
+// serviceRunner runs systemctl; a variable so tests stand in for it.
+var serviceRunner = func(ctx context.Context, args ...string) (string, error) {
+	out, err := exec.CommandContext(ctx, "systemctl", args...).CombinedOutput() //nolint:gosec // fixed command
+	return strings.TrimSpace(string(out)), err
+}
+
+// installedServices are the llmgw units this machine has, or those named.
+func installedServices(ctx context.Context, names []string) ([]string, error) {
+	var want []string
+	for _, n := range names {
+		if !strings.HasPrefix(n, "llmgw-") {
+			n = "llmgw-" + n
+		}
+		want = append(want, n)
+	}
+	var have []string
+	for _, u := range llmgwServices {
+		if _, err := os.Stat(unitDir + "/" + u + ".service"); err != nil {
+			continue
+		}
+		if len(want) > 0 && !hasUnit(want, u) {
+			continue
+		}
+		have = append(have, u)
+	}
+	for _, w := range want {
+		if !hasUnit(have, w) {
+			return nil, fmt.Errorf("%s is not installed here (this machine has: %s)", w, strings.Join(present(), ", "))
+		}
+	}
+	if len(have) == 0 {
+		return nil, errors.New("no llmgw service is installed on this machine")
+	}
+	return have, nil
+}
+
+func present() []string {
+	var out []string
+	for _, u := range llmgwServices {
+		if _, err := os.Stat(unitDir + "/" + u + ".service"); err == nil {
+			out = append(out, strings.TrimPrefix(u, "llmgw-"))
+		}
+	}
+	if len(out) == 0 {
+		return []string{"none"}
+	}
+	return out
+}
+
+func hasUnit(list []string, s string) bool {
+	for _, x := range list {
+		if x == s {
+			return true
+		}
+	}
+	return false
+}
+
+func newServiceCmds() []*cobra.Command {
+	status := &cobra.Command{
+		Use:   "status [server|agent|mirror]...",
+		Short: "Whether llmgw's services on this machine run",
+		Long: "Each llmgw service this machine has — the control plane (server), the\n" +
+			"node agent (agent), the model mirror (mirror) — and whether it runs.\n" +
+			"The cluster as a whole is 'llmgw ctl status'.",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			units, err := localUnits(cmd, args)
+			if err != nil {
+				return err
+			}
+			out := cmd.OutOrStdout()
+			for _, u := range units {
+				state, _ := serviceRunner(cmd.Context(), "is-active", u)
+				since, _ := serviceRunner(cmd.Context(), "show", u, "-p", "ActiveEnterTimestamp", "--value")
+				fmt.Fprintf(out, "%-14s %-10s %s\n", strings.TrimPrefix(u, "llmgw-"), state, since)
+			}
+			return nil
+		},
+	}
+	verb := func(use, short, long, action string, reverse bool) *cobra.Command {
+		var engines bool
+		c := &cobra.Command{
+			Use:   use + " [server|agent|mirror]...",
+			Short: short,
+			Long:  long,
+			RunE: func(cmd *cobra.Command, args []string) error {
+				if os.Geteuid() != 0 {
+					return errors.New("starting and stopping services needs root: run it with sudo")
+				}
+				units, err := localUnits(cmd, args)
+				if err != nil {
+					return err
+				}
+				if reverse {
+					// The agent before the server it dials, the server before
+					// the mirror it reads.
+					for i, j := 0, len(units)-1; i < j; i, j = i+1, j-1 {
+						units[i], units[j] = units[j], units[i]
+					}
+				}
+				out := cmd.OutOrStdout()
+				for _, u := range units {
+					if msg, err := serviceRunner(cmd.Context(), action, u); err != nil {
+						return fmt.Errorf("%s %s: %v %s", action, u, err, msg)
+					}
+					state, _ := serviceRunner(cmd.Context(), "is-active", u)
+					fmt.Fprintf(out, "%-14s %s\n", strings.TrimPrefix(u, "llmgw-"), state)
+				}
+				if action == "stop" && hasUnit(units, "llmgw-agent") {
+					if engines {
+						_, _ = exec.CommandContext(cmd.Context(), "pkill", "-u", "llmgw-engine").CombinedOutput() //nolint:gosec // fixed command
+						fmt.Fprintln(out, "engines   stopped (every process of llmgw-engine)")
+					} else {
+						fmt.Fprintln(out, "\nThe engines keep serving: they outlive the agent, which takes them over again when it starts.\n"+
+							"--engines stops them too; the control plane then places their replicas elsewhere.")
+					}
+				}
+				if action == "stop" && hasUnit(units, "llmgw-server") {
+					fmt.Fprintln(out, "A standby control plane, where there is one, takes over within seconds.")
+				}
+				return nil
+			},
+		}
+		if action == "stop" {
+			c.Flags().BoolVar(&engines, "engines", false, "also stop the engines this node runs")
+		}
+		return c
+	}
+	return []*cobra.Command{
+		status,
+		verb("start", "Start llmgw's services on this machine",
+			"Starts each llmgw service this machine has, or those named: the mirror, the\n"+
+				"control plane, the node agent, in that order.", "start", false),
+		verb("stop", "Stop llmgw's services on this machine",
+			"Stops each llmgw service this machine has, or those named: the agent, the\n"+
+				"control plane, the mirror, in that order. The engines a node runs keep\n"+
+				"serving unless --engines is given; a standby control plane takes over from\n"+
+				"a leader stopped. They stay stopped until 'llmgw start', or the machine\n"+
+				"boots.", "stop", true),
+		verb("restart", "Restart llmgw's services on this machine",
+			"Restarts each llmgw service this machine has, or those named. Engines keep\n"+
+				"serving and are taken over again by the agent; a control plane's settings\n"+
+				"and server.yaml are read anew.", "restart", false),
+	}
+}
+
+func localUnits(cmd *cobra.Command, args []string) ([]string, error) {
+	if runtime.GOOS != "linux" {
+		return nil, fmt.Errorf("llmgw's services run under systemd on Linux; this is %s", runtime.GOOS)
+	}
+	return installedServices(cmd.Context(), args)
+}

internal/cli/service_test.go

@@ -0,0 +1,54 @@
+package cli
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+func TestTheServicesHereAreFoundAndAskedAbout(t *testing.T) {
+	old, oldRun := unitDir, serviceRunner
+	unitDir = t.TempDir()
+	t.Cleanup(func() { unitDir, serviceRunner = old, oldRun })
+	for _, u := range []string{"llmgw-server", "llmgw-agent"} {
+		_ = os.WriteFile(filepath.Join(unitDir, u+".service"), []byte("[Unit]"), 0o644)
+	}
+	var asked []string
+	serviceRunner = func(_ context.Context, args ...string) (string, error) {
+		asked = append(asked, strings.Join(args, " "))
+		if args[0] == "is-active" {
+			return "active", nil
+		}
+		return "Wed 2026-10-07 12:00:00 UTC", nil
+	}
+
+	units, err := installedServices(context.Background(), nil)
+	if err != nil || strings.Join(units, ",") != "llmgw-server,llmgw-agent" {
+		t.Fatalf("%v %v", units, err)
+	}
+	if _, err := installedServices(context.Background(), []string{"mirror"}); err == nil ||
+		!strings.Contains(err.Error(), "server, agent") {
+		t.Errorf("a service not here: %v", err)
+	}
+	if u, err := installedServices(context.Background(), []string{"agent"}); err != nil || len(u) != 1 {
+		t.Errorf("one named: %v %v", u, err)
+	}
+
+	if runtime.GOOS != "linux" {
+		return
+	}
+	var out bytes.Buffer
+	c := newServiceCmds()[0]
+	c.SetOut(&out)
+	c.SetContext(context.Background())
+	if err := c.RunE(c, nil); err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(out.String(), "server") || !strings.Contains(out.String(), "active") {
+		t.Errorf("%s", out.String())
+	}
+}