diff --git a/internal/cli/system.go b/internal/cli/system.go index 091649e..7de9502 100644 --- a/internal/cli/system.go +++ b/internal/cli/system.go @@ -151,6 +151,9 @@ func newSystemCmds() []*cobra.Command { others = append(others, cp) } } + // One that cannot be reached is said and left: the rest of the + // system still restarts, and the leader still goes last. + var failed []string for pass := 0; pass < 2; pass++ { for _, cp := range others { leading := cp.State == "leading" @@ -159,8 +162,9 @@ func newSystemCmds() []*cobra.Command { } r := &onboard.SSH{Addr: cp.Address, User: sshUser} if res, err := r.Run(ctx, "sudo systemctl restart llmgw-server", nil); err != nil { - return fmt.Errorf("%s (%s) could not be restarted over ssh as %q: %v %s; "+ - "run `sudo llmgw service restart server` there", cp.Name, cp.Address, sshUser, err, strings.TrimSpace(string(res))) + fmt.Fprintf(out, "%-10s NOT restarted: ssh as %q: %v %s\n", cp.Name, sshUser, err, strings.TrimSpace(string(res))) + failed = append(failed, cp.Name+" ("+cp.Address+")") + continue } fmt.Fprintf(out, "%-10s restarted (%s)\n", cp.Name, cp.State) time.Sleep(5 * time.Second) @@ -174,6 +178,10 @@ func newSystemCmds() []*cobra.Command { fmt.Fprintln(out, "this restarted") } fmt.Fprintln(out, "\nllmgw status shows the nodes coming back; their replicas are taken over as they are.") + if len(failed) > 0 { + return fmt.Errorf("not restarted: %s; run `sudo llmgw service restart server` there, "+ + "or give --ssh-user someone this machine's root can log in as", strings.Join(failed, ", ")) + } return nil }, })