From 3f8cc06f722786acda1abe79b3291af9650ea823 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 7 Oct 2026 18:55:52 +0200 Subject: [PATCH] llmgw restart goes on past a control plane it cannot reach One standby that ssh could not log in to stopped the whole restart before this control plane's own. It is now said and left, the rest restarts - the leader last - and the command fails at the end naming what was not restarted and how to do it there. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/cli/system.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/internal/cli/system.go b/internal/cli/system.go index 091649e..7de9502 100644 --- a/internal/cli/system.go +++ b/internal/cli/system.go @@ -151,6 +151,9 @@ func newSystemCmds() []*cobra.Command { others = append(others, cp) } } + // One that cannot be reached is said and left: the rest of the + // system still restarts, and the leader still goes last. + var failed []string for pass := 0; pass < 2; pass++ { for _, cp := range others { leading := cp.State == "leading" @@ -159,8 +162,9 @@ func newSystemCmds() []*cobra.Command { } r := &onboard.SSH{Addr: cp.Address, User: sshUser} if res, err := r.Run(ctx, "sudo systemctl restart llmgw-server", nil); err != nil { - return fmt.Errorf("%s (%s) could not be restarted over ssh as %q: %v %s; "+ - "run `sudo llmgw service restart server` there", cp.Name, cp.Address, sshUser, err, strings.TrimSpace(string(res))) + fmt.Fprintf(out, "%-10s NOT restarted: ssh as %q: %v %s\n", cp.Name, sshUser, err, strings.TrimSpace(string(res))) + failed = append(failed, cp.Name+" ("+cp.Address+")") + continue } fmt.Fprintf(out, "%-10s restarted (%s)\n", cp.Name, cp.State) time.Sleep(5 * time.Second) @@ -174,6 +178,10 @@ func newSystemCmds() []*cobra.Command { fmt.Fprintln(out, "this restarted") } fmt.Fprintln(out, "\nllmgw status shows the nodes coming back; their replicas are taken over as they are.") + if len(failed) > 0 { + return fmt.Errorf("not restarted: %s; run `sudo llmgw service restart server` there, "+ + "or give --ssh-user someone this machine's root can log in as", strings.Join(failed, ", ")) + } return nil }, })