llmgw restart goes on past a control plane it cannot reach

One standby that ssh could not log in to stopped the whole restart before
this control plane's own. It is now said and left, the rest restarts - the
leader last - and the command fails at the end naming what was not
restarted and how to do it there.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-07 18:55:52 +02:00
1 parent 7245dbe1dd
commit 3f8cc06f72
1 file changed
+10 -2
+10 -2
View File
@@ -151,6 +151,9 @@ func newSystemCmds() []*cobra.Command {
others = append(others, cp)
}
}
// One that cannot be reached is said and left: the rest of the
// system still restarts, and the leader still goes last.
var failed []string
for pass := 0; pass < 2; pass++ {
for _, cp := range others {
leading := cp.State == "leading"
@@ -159,8 +162,9 @@ func newSystemCmds() []*cobra.Command {
}
r := &onboard.SSH{Addr: cp.Address, User: sshUser}
if res, err := r.Run(ctx, "sudo systemctl restart llmgw-server", nil); err != nil {
return fmt.Errorf("%s (%s) could not be restarted over ssh as %q: %v %s; "+
"run `sudo llmgw service restart server` there", cp.Name, cp.Address, sshUser, err, strings.TrimSpace(string(res)))
fmt.Fprintf(out, "%-10s NOT restarted: ssh as %q: %v %s\n", cp.Name, sshUser, err, strings.TrimSpace(string(res)))
failed = append(failed, cp.Name+" ("+cp.Address+")")
continue
}
fmt.Fprintf(out, "%-10s restarted (%s)\n", cp.Name, cp.State)
time.Sleep(5 * time.Second)
@@ -174,6 +178,10 @@ func newSystemCmds() []*cobra.Command {
fmt.Fprintln(out, "this restarted")
}
fmt.Fprintln(out, "\nllmgw status shows the nodes coming back; their replicas are taken over as they are.")
if len(failed) > 0 {
return fmt.Errorf("not restarted: %s; run `sudo llmgw service restart server` there, "+
"or give --ssh-user someone this machine's root can log in as", strings.Join(failed, ", "))
}
return nil
},
})