ctl upgrade asks the server what it runs, and waits for a node as long as it does

A binary installed by hand made ctl upgrade say this machine already ran
the release and leave the old server running. And a node had 15 seconds
to answer an upgrade the control plane gives five minutes, so a node that
was upgrading stopped the run.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 12:43:25 +02:00
1 parent dc65ea6899
commit 4ff62c5dd2
6 files changed
+77 -10

No files matched your search

+8
View File
@@ -39,6 +39,14 @@ func newCtlClient(addr string, tokenFiles ...string) *ctlClient {
}
}
// patient is the same client waiting up to d for an answer, for a call the
// server itself may spend minutes on.
func (c *ctlClient) patient(d time.Duration) *ctlClient {
cp := *c
cp.http = &http.Client{Timeout: d}
return &cp
}
func (c *ctlClient) get(ctx context.Context, path string, out any) error {
return c.do(ctx, http.MethodGet, path, nil, out)
}
+30 -7
View File
@@ -152,8 +152,20 @@ func upgradeOtherControlPlanes(ctx context.Context, out io.Writer, client func()
return nil
}
// controlPlaneStep is what this machine needs to run v: the binary installed,
// the service restarted, both, or neither. The binary is this command's own;
// running is what the server says it runs, "" from one too old to say. A
// binary put in place by hand is not a server running it.
func controlPlaneStep(binary, running, v string) (install, restart bool) {
install = binary != v
return install, install || running != v
}
func upgradeControlPlane(ctx context.Context, out io.Writer, client func() *ctlClient, v, releases, svc string) error {
if version.Version == v {
var sum adminapi.Summary
_ = client().get(ctx, "/api/v1/summary", &sum)
install, restart := controlPlaneStep(version.Version, sum.Version, v)
if !restart {
fmt.Fprintf(out, "this machine already runs %s\n", v)
return nil
}
@@ -165,12 +177,20 @@ func upgradeControlPlane(ctx context.Context, out io.Writer, client func() *ctlC
if err != nil {
return err
}
fmt.Fprintf(out, "== this machine: %s -> %s\n", version.Version, v)
backup, err := release.Install(ctx, nil, releases, v, runtime.GOOS, runtime.GOARCH, self)
if err != nil {
return err
backup := "the binary installed by hand before it"
if install {
fmt.Fprintf(out, "== this machine: %s -> %s\n", version.Version, v)
if backup, err = release.Install(ctx, nil, releases, v, runtime.GOOS, runtime.GOARCH, self); err != nil {
return err
}
fmt.Fprintf(out, " installed, checksum verified; the old binary is %s\n", backup)
} else {
running := sum.Version
if running == "" {
running = "an older release"
}
fmt.Fprintf(out, "== this machine: %s is installed, and %s still runs it; restarting %s\n", v, running, svc)
}
fmt.Fprintf(out, " installed, checksum verified; the old binary is %s\n", backup)
var before []adminapi.NodeView
_ = client().get(ctx, "/api/v1/nodes", &before)
if b, err := exec.CommandContext(ctx, "systemctl", "restart", svc).CombinedOutput(); err != nil { //nolint:gosec // fixed command
@@ -270,7 +290,10 @@ func upgradeWave(ctx context.Context, out io.Writer, client func() *ctlClient, v
body, _ := json.Marshal(adminapi.UpgradeRequest{Version: v})
var failed []string
for _, n := range wave {
if err := client().post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil {
// The control plane waits for the agent to fetch and check the
// release, up to five minutes; fifteen seconds gave up on a node
// that was upgrading.
if err := client().patient(6*time.Minute).post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil {
failed = append(failed, fmt.Sprintf("%s: %v", n.ID, err))
}
}
+19
View File
@@ -131,3 +131,22 @@ func TestAnUpgradeKnowsWhetherItRunsOnTheLeader(t *testing.T) {
t.Error("the leader took itself for a standby")
}
}
func TestABinaryInstalledByHandIsNotAServerRunningIt(t *testing.T) {
// The binary put in place with install, llmgw-server not restarted: the
// upgrade said "already runs" and left the old server running.
for _, c := range []struct {
binary, running string
install, restart bool
}{
{"v1.0.0", "v1.0.0", true, true}, // the usual: fetch it, restart
{"v1.0.1", "v1.0.0", false, true}, // installed by hand: restart only
{"v1.0.1", "", false, true}, // a server too old to say
{"v1.0.1", "v1.0.1", false, false},
} {
install, restart := controlPlaneStep(c.binary, c.running, "v1.0.1")
if install != c.install || restart != c.restart {
t.Errorf("binary %s, running %q: install %v restart %v", c.binary, c.running, install, restart)
}
}
}