From ef33e52d3d287510994d848d6c2253ad67b979be Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Mon, 5 Oct 2026 06:45:56 +0200 Subject: [PATCH] ctl upgrade upgrades the leader last, wherever it is run Run on the standby, it upgraded the leader first, then itself, and the leadership moved twice. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 2 ++ docs/install.md | 9 +++---- internal/cli/ctl_upgrade.go | 41 +++++++++++++++++++++++++------- internal/cli/ctl_upgrade_test.go | 20 ++++++++++++++++ 4 files changed, 60 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b80abf3..2ea4fee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,8 @@ Two control planes, run in anger on the reference cluster. ### Fixed +- `ctl upgrade` upgrades the leader last wherever it is run; run on the + standby, it upgraded the leader first and the leadership moved twice. - **Failover took 40 seconds; it takes a few.** A stopping control plane held every agent's stream for the thirty seconds of a graceful shutdown; it now gives up its lease first and lets the agents go at once. diff --git a/docs/install.md b/docs/install.md index 27f1e30..2264c2f 100644 --- a/docs/install.md +++ b/docs/install.md @@ -2259,12 +2259,13 @@ On a control plane: sudo --preserve-env=SSH_AUTH_SOCK llmgw ctl upgrade v0.10.4 ``` -With two control planes it upgrades the other one first, over ssh as the user -who ran sudo (`--ssh-user`), with `llmgw upgrade` there; then this machine — +With two control planes it upgrades the standby first and the leader last, +whichever this machine is — the other over ssh as the user who ran sudo +(`--ssh-user`), with `llmgw upgrade` there, and this machine — fetching the release for its platform, checking it against the release's checksums, keeping the old binary as `/usr/local/bin/llmgw.bak-pre-`, -restarting `llmgw-server` and waiting for every node to reconnect. A leader -restarted last hands over to a standby already on the release. +restarting `llmgw-server` and waiting for every node to reconnect. The +leader's restart hands over once, to a standby already on the release. Then the nodes, in waves: one first, then two, four, and so on up to `--parallel` (20) at once. A node's agent is told the version over the stream diff --git a/internal/cli/ctl_upgrade.go b/internal/cli/ctl_upgrade.go index c2cbce8..af86364 100644 --- a/internal/cli/ctl_upgrade.go +++ b/internal/cli/ctl_upgrade.go @@ -68,14 +68,26 @@ func newUpgradeCmd(client func() *ctlClient) *cobra.Command { } } if !skipCP { - if othersCP { - if err := upgradeOtherControlPlanes(ctx, out, client, v, releases, sshUser); err != nil { + // The leader last, wherever this runs: every other control + // plane is a standby, upgraded with nothing moving, and the + // leader's restart hands over once, to a machine already on + // the release. + others := func() error { + if !othersCP { + return nil + } + return upgradeOtherControlPlanes(ctx, out, client, v, releases, sshUser) + } + self := func() error { return upgradeControlPlane(ctx, out, client, v, releases, serverSvc) } + order := []func() error{others, self} + if !thisLeads(ctx, client) { + order = []func() error{self, others} + } + for _, step := range order { + if err := step(); err != nil { return err } } - if err := upgradeControlPlane(ctx, out, client, v, releases, serverSvc); err != nil { - return err - } } return upgradeNodes(ctx, out, client, v, nodes, perNode, parallel) }, @@ -92,10 +104,23 @@ func newUpgradeCmd(client func() *ctlClient) *cobra.Command { return c } +// thisLeads says whether this machine's control plane leads; one that does +// not say, alone or older, is taken to. +func thisLeads(ctx context.Context, client func() *ctlClient) bool { + var sum adminapi.Summary + if err := client().get(ctx, "/api/v1/summary", &sum); err != nil { + return true + } + for _, cp := range sum.ControlPlanes.Members { + if cp.This { + return cp.State == "leading" + } + } + return true +} + // upgradeOtherControlPlanes moves every other control plane of the cluster to -// v, over ssh with `llmgw upgrade`, before this machine: each is a standby, or -// becomes one when this one is restarted, and the leadership should move to -// a machine already on the release. Nothing is changed here if one cannot be +// v, over ssh with `llmgw upgrade`. Nothing is changed here if one cannot be // reached. func upgradeOtherControlPlanes(ctx context.Context, out io.Writer, client func() *ctlClient, v, releases, user string) error { var sum adminapi.Summary diff --git a/internal/cli/ctl_upgrade_test.go b/internal/cli/ctl_upgrade_test.go index 94d7bc2..bcfda52 100644 --- a/internal/cli/ctl_upgrade_test.go +++ b/internal/cli/ctl_upgrade_test.go @@ -111,3 +111,23 @@ func TestAnUpgradeGoesInWavesThatGrowUpToTheLimit(t *testing.T) { t.Errorf("waves %s:\n%s", got, out.String()) } } + +func TestAnUpgradeKnowsWhetherItRunsOnTheLeader(t *testing.T) { + sum := adminapi.Summary{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(sum) + })) + defer srv.Close() + client := func() *ctlClient { return newCtlClient(srv.URL) } + if !thisLeads(t.Context(), client) { + t.Error("a control plane with no rival does not lead") + } + sum.ControlPlanes.Members = []adminapi.ControlPlane{{Name: "a", State: "leading"}, {Name: "b", State: "standing by", This: true}} + if thisLeads(t.Context(), client) { + t.Error("a standby took itself for the leader: it would upgrade the leader before itself") + } + sum.ControlPlanes.Members[0].This, sum.ControlPlanes.Members[1].This = true, false + if !thisLeads(t.Context(), client) { + t.Error("the leader took itself for a standby") + } +}