ctl upgrade asks the server what it runs, and waits for a node as long as it does
A binary installed by hand made ctl upgrade say this machine already ran the release and leave the old server running. And a node had 15 seconds to answer an upgrade the control plane gives five minutes, so a node that was upgrading stopped the run. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
dc65ea6899
commit
4ff62c5dd2
6 files changed
+77
-10
No files matched your search
@@ -39,6 +39,14 @@ func newCtlClient(addr string, tokenFiles ...string) *ctlClient {
|
||||
}
|
||||
}
|
||||
|
||||
// patient is the same client waiting up to d for an answer, for a call the
|
||||
// server itself may spend minutes on.
|
||||
func (c *ctlClient) patient(d time.Duration) *ctlClient {
|
||||
cp := *c
|
||||
cp.http = &http.Client{Timeout: d}
|
||||
return &cp
|
||||
}
|
||||
|
||||
func (c *ctlClient) get(ctx context.Context, path string, out any) error {
|
||||
return c.do(ctx, http.MethodGet, path, nil, out)
|
||||
}
|
||||
|
||||
@@ -152,8 +152,20 @@ func upgradeOtherControlPlanes(ctx context.Context, out io.Writer, client func()
|
||||
return nil
|
||||
}
|
||||
|
||||
// controlPlaneStep is what this machine needs to run v: the binary installed,
|
||||
// the service restarted, both, or neither. The binary is this command's own;
|
||||
// running is what the server says it runs, "" from one too old to say. A
|
||||
// binary put in place by hand is not a server running it.
|
||||
func controlPlaneStep(binary, running, v string) (install, restart bool) {
|
||||
install = binary != v
|
||||
return install, install || running != v
|
||||
}
|
||||
|
||||
func upgradeControlPlane(ctx context.Context, out io.Writer, client func() *ctlClient, v, releases, svc string) error {
|
||||
if version.Version == v {
|
||||
var sum adminapi.Summary
|
||||
_ = client().get(ctx, "/api/v1/summary", &sum)
|
||||
install, restart := controlPlaneStep(version.Version, sum.Version, v)
|
||||
if !restart {
|
||||
fmt.Fprintf(out, "this machine already runs %s\n", v)
|
||||
return nil
|
||||
}
|
||||
@@ -165,12 +177,20 @@ func upgradeControlPlane(ctx context.Context, out io.Writer, client func() *ctlC
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, "== this machine: %s -> %s\n", version.Version, v)
|
||||
backup, err := release.Install(ctx, nil, releases, v, runtime.GOOS, runtime.GOARCH, self)
|
||||
if err != nil {
|
||||
return err
|
||||
backup := "the binary installed by hand before it"
|
||||
if install {
|
||||
fmt.Fprintf(out, "== this machine: %s -> %s\n", version.Version, v)
|
||||
if backup, err = release.Install(ctx, nil, releases, v, runtime.GOOS, runtime.GOARCH, self); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, " installed, checksum verified; the old binary is %s\n", backup)
|
||||
} else {
|
||||
running := sum.Version
|
||||
if running == "" {
|
||||
running = "an older release"
|
||||
}
|
||||
fmt.Fprintf(out, "== this machine: %s is installed, and %s still runs it; restarting %s\n", v, running, svc)
|
||||
}
|
||||
fmt.Fprintf(out, " installed, checksum verified; the old binary is %s\n", backup)
|
||||
var before []adminapi.NodeView
|
||||
_ = client().get(ctx, "/api/v1/nodes", &before)
|
||||
if b, err := exec.CommandContext(ctx, "systemctl", "restart", svc).CombinedOutput(); err != nil { //nolint:gosec // fixed command
|
||||
@@ -270,7 +290,10 @@ func upgradeWave(ctx context.Context, out io.Writer, client func() *ctlClient, v
|
||||
body, _ := json.Marshal(adminapi.UpgradeRequest{Version: v})
|
||||
var failed []string
|
||||
for _, n := range wave {
|
||||
if err := client().post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil {
|
||||
// The control plane waits for the agent to fetch and check the
|
||||
// release, up to five minutes; fifteen seconds gave up on a node
|
||||
// that was upgrading.
|
||||
if err := client().patient(6*time.Minute).post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil {
|
||||
failed = append(failed, fmt.Sprintf("%s: %v", n.ID, err))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,3 +131,22 @@ func TestAnUpgradeKnowsWhetherItRunsOnTheLeader(t *testing.T) {
|
||||
t.Error("the leader took itself for a standby")
|
||||
}
|
||||
}
|
||||
|
||||
func TestABinaryInstalledByHandIsNotAServerRunningIt(t *testing.T) {
|
||||
// The binary put in place with install, llmgw-server not restarted: the
|
||||
// upgrade said "already runs" and left the old server running.
|
||||
for _, c := range []struct {
|
||||
binary, running string
|
||||
install, restart bool
|
||||
}{
|
||||
{"v1.0.0", "v1.0.0", true, true}, // the usual: fetch it, restart
|
||||
{"v1.0.1", "v1.0.0", false, true}, // installed by hand: restart only
|
||||
{"v1.0.1", "", false, true}, // a server too old to say
|
||||
{"v1.0.1", "v1.0.1", false, false},
|
||||
} {
|
||||
install, restart := controlPlaneStep(c.binary, c.running, "v1.0.1")
|
||||
if install != c.install || restart != c.restart {
|
||||
t.Errorf("binary %s, running %q: install %v restart %v", c.binary, c.running, install, restart)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user