llmgw restart goes on past a control plane it cannot reach
One standby that ssh could not log in to stopped the whole restart before this control plane's own. It is now said and left, the rest restarts - the leader last - and the command fails at the end naming what was not restarted and how to do it there. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
7245dbe1dd
commit
3f8cc06f72
1 file changed
+10
-2
+10
-2
@@ -151,6 +151,9 @@ func newSystemCmds() []*cobra.Command {
|
||||
others = append(others, cp)
|
||||
}
|
||||
}
|
||||
// One that cannot be reached is said and left: the rest of the
|
||||
// system still restarts, and the leader still goes last.
|
||||
var failed []string
|
||||
for pass := 0; pass < 2; pass++ {
|
||||
for _, cp := range others {
|
||||
leading := cp.State == "leading"
|
||||
@@ -159,8 +162,9 @@ func newSystemCmds() []*cobra.Command {
|
||||
}
|
||||
r := &onboard.SSH{Addr: cp.Address, User: sshUser}
|
||||
if res, err := r.Run(ctx, "sudo systemctl restart llmgw-server", nil); err != nil {
|
||||
return fmt.Errorf("%s (%s) could not be restarted over ssh as %q: %v %s; "+
|
||||
"run `sudo llmgw service restart server` there", cp.Name, cp.Address, sshUser, err, strings.TrimSpace(string(res)))
|
||||
fmt.Fprintf(out, "%-10s NOT restarted: ssh as %q: %v %s\n", cp.Name, sshUser, err, strings.TrimSpace(string(res)))
|
||||
failed = append(failed, cp.Name+" ("+cp.Address+")")
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(out, "%-10s restarted (%s)\n", cp.Name, cp.State)
|
||||
time.Sleep(5 * time.Second)
|
||||
@@ -174,6 +178,10 @@ func newSystemCmds() []*cobra.Command {
|
||||
fmt.Fprintln(out, "this restarted")
|
||||
}
|
||||
fmt.Fprintln(out, "\nllmgw status shows the nodes coming back; their replicas are taken over as they are.")
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("not restarted: %s; run `sudo llmgw service restart server` there, "+
|
||||
"or give --ssh-user someone this machine's root can log in as", strings.Join(failed, ", "))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
Reference in new issue
Block a user