From 05aa21c4d49dc16e9e9c7e4543490652845f4c62 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 7 Oct 2026 14:52:52 +0200 Subject: [PATCH] llmgw status, start, stop and restart for this machine's services There was no command to stop or start llmgw on a machine (issue #3). These act on the llmgw units the machine has - mirror, server, agent - or those named, in dependency order, saying what each did. stop leaves a node's engines serving unless --engines. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 4 + docs/install.md | 15 +++ internal/cli/root.go | 2 + internal/cli/service.go | 174 +++++++++++++++++++++++++++++++++++ internal/cli/service_test.go | 54 +++++++++++ 5 files changed, 249 insertions(+) create mode 100644 internal/cli/service.go create mode 100644 internal/cli/service_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f16b1c47..0b8723da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,10 @@ What was left of the field report from v0.10.6. ### Added +- **`llmgw status`, `start`, `stop` and `restart`** for this machine's llmgw + services — server, agent, mirror, or those named — in the order they + depend on each other. Stopping the agent leaves the engines serving unless + `--engines` is given; stopping a leader hands over to a standby. - **A replica says what it computes on**: `ctl replicas` (ON), the console and the admin API show `CUDA` with the GPUs its processes hold memory on and how much, as nvidia-smi sees them, or `CPU` — for every engine, kept by diff --git a/docs/install.md b/docs/install.md index 9d5d780e..e8463364 100644 --- a/docs/install.md +++ b/docs/install.md @@ -2294,6 +2294,21 @@ A cluster set up online becomes offline the same way: `bundle import` on the mirror and control plane, then the `offline:` block in `server.yaml` and a restart. +## Stopping and starting a machine + +``` +sudo llmgw status # each llmgw service here, and whether it runs +sudo llmgw stop # agent, control plane, mirror — whichever this machine has +sudo llmgw start +sudo llmgw restart agent # or name one: server, agent, mirror +``` + +Stopping a node's agent leaves its engines serving — the agent takes them +over again when it starts — unless `llmgw stop --engines` stops them too, and +the control plane then places their replicas elsewhere. Stopping the leading +control plane hands over to a standby within seconds. The whole cluster's +state is `llmgw ctl status`. + ## Removing llmgw from a machine ``` diff --git a/internal/cli/root.go b/internal/cli/root.go index 0414b9e6..ff3db3b3 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -63,6 +63,8 @@ func Root() *cobra.Command { newWipeCmd(), newDocsCmd(), ) + // status, start, stop and restart: this machine's llmgw services. + root.AddCommand(newServiceCmds()...) return root } diff --git a/internal/cli/service.go b/internal/cli/service.go new file mode 100644 index 00000000..6f54ee13 --- /dev/null +++ b/internal/cli/service.go @@ -0,0 +1,174 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "runtime" + "strings" + + "github.com/spf13/cobra" +) + +// llmgwServices are the units llmgw runs as on a machine, in the order they +// start: the store's clients after the store, the server before the agent +// that dials it. +var llmgwServices = []string{"llmgw-mirror", "llmgw-server", "llmgw-agent"} + +// unitDir is where llmgw's units are installed. +var unitDir = "/etc/systemd/system" + +// serviceRunner runs systemctl; a variable so tests stand in for it. +var serviceRunner = func(ctx context.Context, args ...string) (string, error) { + out, err := exec.CommandContext(ctx, "systemctl", args...).CombinedOutput() //nolint:gosec // fixed command + return strings.TrimSpace(string(out)), err +} + +// installedServices are the llmgw units this machine has, or those named. +func installedServices(ctx context.Context, names []string) ([]string, error) { + var want []string + for _, n := range names { + if !strings.HasPrefix(n, "llmgw-") { + n = "llmgw-" + n + } + want = append(want, n) + } + var have []string + for _, u := range llmgwServices { + if _, err := os.Stat(unitDir + "/" + u + ".service"); err != nil { + continue + } + if len(want) > 0 && !hasUnit(want, u) { + continue + } + have = append(have, u) + } + for _, w := range want { + if !hasUnit(have, w) { + return nil, fmt.Errorf("%s is not installed here (this machine has: %s)", w, strings.Join(present(), ", ")) + } + } + if len(have) == 0 { + return nil, errors.New("no llmgw service is installed on this machine") + } + return have, nil +} + +func present() []string { + var out []string + for _, u := range llmgwServices { + if _, err := os.Stat(unitDir + "/" + u + ".service"); err == nil { + out = append(out, strings.TrimPrefix(u, "llmgw-")) + } + } + if len(out) == 0 { + return []string{"none"} + } + return out +} + +func hasUnit(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +func newServiceCmds() []*cobra.Command { + status := &cobra.Command{ + Use: "status [server|agent|mirror]...", + Short: "Whether llmgw's services on this machine run", + Long: "Each llmgw service this machine has — the control plane (server), the\n" + + "node agent (agent), the model mirror (mirror) — and whether it runs.\n" + + "The cluster as a whole is 'llmgw ctl status'.", + RunE: func(cmd *cobra.Command, args []string) error { + units, err := localUnits(cmd, args) + if err != nil { + return err + } + out := cmd.OutOrStdout() + for _, u := range units { + state, _ := serviceRunner(cmd.Context(), "is-active", u) + since, _ := serviceRunner(cmd.Context(), "show", u, "-p", "ActiveEnterTimestamp", "--value") + fmt.Fprintf(out, "%-14s %-10s %s\n", strings.TrimPrefix(u, "llmgw-"), state, since) + } + return nil + }, + } + verb := func(use, short, long, action string, reverse bool) *cobra.Command { + var engines bool + c := &cobra.Command{ + Use: use + " [server|agent|mirror]...", + Short: short, + Long: long, + RunE: func(cmd *cobra.Command, args []string) error { + if os.Geteuid() != 0 { + return errors.New("starting and stopping services needs root: run it with sudo") + } + units, err := localUnits(cmd, args) + if err != nil { + return err + } + if reverse { + // The agent before the server it dials, the server before + // the mirror it reads. + for i, j := 0, len(units)-1; i < j; i, j = i+1, j-1 { + units[i], units[j] = units[j], units[i] + } + } + out := cmd.OutOrStdout() + for _, u := range units { + if msg, err := serviceRunner(cmd.Context(), action, u); err != nil { + return fmt.Errorf("%s %s: %v %s", action, u, err, msg) + } + state, _ := serviceRunner(cmd.Context(), "is-active", u) + fmt.Fprintf(out, "%-14s %s\n", strings.TrimPrefix(u, "llmgw-"), state) + } + if action == "stop" && hasUnit(units, "llmgw-agent") { + if engines { + _, _ = exec.CommandContext(cmd.Context(), "pkill", "-u", "llmgw-engine").CombinedOutput() //nolint:gosec // fixed command + fmt.Fprintln(out, "engines stopped (every process of llmgw-engine)") + } else { + fmt.Fprintln(out, "\nThe engines keep serving: they outlive the agent, which takes them over again when it starts.\n"+ + "--engines stops them too; the control plane then places their replicas elsewhere.") + } + } + if action == "stop" && hasUnit(units, "llmgw-server") { + fmt.Fprintln(out, "A standby control plane, where there is one, takes over within seconds.") + } + return nil + }, + } + if action == "stop" { + c.Flags().BoolVar(&engines, "engines", false, "also stop the engines this node runs") + } + return c + } + return []*cobra.Command{ + status, + verb("start", "Start llmgw's services on this machine", + "Starts each llmgw service this machine has, or those named: the mirror, the\n"+ + "control plane, the node agent, in that order.", "start", false), + verb("stop", "Stop llmgw's services on this machine", + "Stops each llmgw service this machine has, or those named: the agent, the\n"+ + "control plane, the mirror, in that order. The engines a node runs keep\n"+ + "serving unless --engines is given; a standby control plane takes over from\n"+ + "a leader stopped. They stay stopped until 'llmgw start', or the machine\n"+ + "boots.", "stop", true), + verb("restart", "Restart llmgw's services on this machine", + "Restarts each llmgw service this machine has, or those named. Engines keep\n"+ + "serving and are taken over again by the agent; a control plane's settings\n"+ + "and server.yaml are read anew.", "restart", false), + } +} + +func localUnits(cmd *cobra.Command, args []string) ([]string, error) { + if runtime.GOOS != "linux" { + return nil, fmt.Errorf("llmgw's services run under systemd on Linux; this is %s", runtime.GOOS) + } + return installedServices(cmd.Context(), args) +} diff --git a/internal/cli/service_test.go b/internal/cli/service_test.go new file mode 100644 index 00000000..95930404 --- /dev/null +++ b/internal/cli/service_test.go @@ -0,0 +1,54 @@ +package cli + +import ( + "bytes" + "context" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestTheServicesHereAreFoundAndAskedAbout(t *testing.T) { + old, oldRun := unitDir, serviceRunner + unitDir = t.TempDir() + t.Cleanup(func() { unitDir, serviceRunner = old, oldRun }) + for _, u := range []string{"llmgw-server", "llmgw-agent"} { + _ = os.WriteFile(filepath.Join(unitDir, u+".service"), []byte("[Unit]"), 0o644) + } + var asked []string + serviceRunner = func(_ context.Context, args ...string) (string, error) { + asked = append(asked, strings.Join(args, " ")) + if args[0] == "is-active" { + return "active", nil + } + return "Wed 2026-10-07 12:00:00 UTC", nil + } + + units, err := installedServices(context.Background(), nil) + if err != nil || strings.Join(units, ",") != "llmgw-server,llmgw-agent" { + t.Fatalf("%v %v", units, err) + } + if _, err := installedServices(context.Background(), []string{"mirror"}); err == nil || + !strings.Contains(err.Error(), "server, agent") { + t.Errorf("a service not here: %v", err) + } + if u, err := installedServices(context.Background(), []string{"agent"}); err != nil || len(u) != 1 { + t.Errorf("one named: %v %v", u, err) + } + + if runtime.GOOS != "linux" { + return + } + var out bytes.Buffer + c := newServiceCmds()[0] + c.SetOut(&out) + c.SetContext(context.Background()) + if err := c.RunE(c, nil); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "server") || !strings.Contains(out.String(), "active") { + t.Errorf("%s", out.String()) + } +}