ctl upgrade upgrades the leader last, wherever it is run

Run on the standby, it upgraded the leader first, then itself, and the
leadership moved twice.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 06:45:56 +02:00
1 parent abaf086fd5
commit ef33e52d3d
4 files changed
+60 -12

No files matched your search

+2
View File
@@ -55,6 +55,8 @@ Two control planes, run in anger on the reference cluster.
### Fixed
- `ctl upgrade` upgrades the leader last wherever it is run; run on the
standby, it upgraded the leader first and the leadership moved twice.
- **Failover took 40 seconds; it takes a few.** A stopping control plane held
every agent's stream for the thirty seconds of a graceful shutdown; it now
gives up its lease first and lets the agents go at once.
+5 -4
View File
@@ -2259,12 +2259,13 @@ On a control plane:
sudo --preserve-env=SSH_AUTH_SOCK llmgw ctl upgrade v0.10.4
```
With two control planes it upgrades the other one first, over ssh as the user
who ran sudo (`--ssh-user`), with `llmgw upgrade` there; then this machine —
With two control planes it upgrades the standby first and the leader last,
whichever this machine is — the other over ssh as the user who ran sudo
(`--ssh-user`), with `llmgw upgrade` there, and this machine —
fetching the release for its platform, checking it against the release's
checksums, keeping the old binary as `/usr/local/bin/llmgw.bak-pre-<version>`,
restarting `llmgw-server` and waiting for every node to reconnect. A leader
restarted last hands over to a standby already on the release.
restarting `llmgw-server` and waiting for every node to reconnect. The
leader's restart hands over once, to a standby already on the release.
Then the nodes, in waves: one first, then two, four, and so on up to
`--parallel` (20) at once. A node's agent is told the version over the stream
+33 -8
View File
@@ -68,14 +68,26 @@ func newUpgradeCmd(client func() *ctlClient) *cobra.Command {
}
}
if !skipCP {
if othersCP {
if err := upgradeOtherControlPlanes(ctx, out, client, v, releases, sshUser); err != nil {
// The leader last, wherever this runs: every other control
// plane is a standby, upgraded with nothing moving, and the
// leader's restart hands over once, to a machine already on
// the release.
others := func() error {
if !othersCP {
return nil
}
return upgradeOtherControlPlanes(ctx, out, client, v, releases, sshUser)
}
self := func() error { return upgradeControlPlane(ctx, out, client, v, releases, serverSvc) }
order := []func() error{others, self}
if !thisLeads(ctx, client) {
order = []func() error{self, others}
}
for _, step := range order {
if err := step(); err != nil {
return err
}
}
if err := upgradeControlPlane(ctx, out, client, v, releases, serverSvc); err != nil {
return err
}
}
return upgradeNodes(ctx, out, client, v, nodes, perNode, parallel)
},
@@ -92,10 +104,23 @@ func newUpgradeCmd(client func() *ctlClient) *cobra.Command {
return c
}
// thisLeads says whether this machine's control plane leads; one that does
// not say, alone or older, is taken to.
func thisLeads(ctx context.Context, client func() *ctlClient) bool {
var sum adminapi.Summary
if err := client().get(ctx, "/api/v1/summary", &sum); err != nil {
return true
}
for _, cp := range sum.ControlPlanes.Members {
if cp.This {
return cp.State == "leading"
}
}
return true
}
// upgradeOtherControlPlanes moves every other control plane of the cluster to
// v, over ssh with `llmgw upgrade`, before this machine: each is a standby, or
// becomes one when this one is restarted, and the leadership should move to
// a machine already on the release. Nothing is changed here if one cannot be
// v, over ssh with `llmgw upgrade`. Nothing is changed here if one cannot be
// reached.
func upgradeOtherControlPlanes(ctx context.Context, out io.Writer, client func() *ctlClient, v, releases, user string) error {
var sum adminapi.Summary
+20
View File
@@ -111,3 +111,23 @@ func TestAnUpgradeGoesInWavesThatGrowUpToTheLimit(t *testing.T) {
t.Errorf("waves %s:\n%s", got, out.String())
}
}
func TestAnUpgradeKnowsWhetherItRunsOnTheLeader(t *testing.T) {
sum := adminapi.Summary{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_ = json.NewEncoder(w).Encode(sum)
}))
defer srv.Close()
client := func() *ctlClient { return newCtlClient(srv.URL) }
if !thisLeads(t.Context(), client) {
t.Error("a control plane with no rival does not lead")
}
sum.ControlPlanes.Members = []adminapi.ControlPlane{{Name: "a", State: "leading"}, {Name: "b", State: "standing by", This: true}}
if thisLeads(t.Context(), client) {
t.Error("a standby took itself for the leader: it would upgrade the leader before itself")
}
sum.ControlPlanes.Members[0].This, sum.ControlPlanes.Members[1].This = true, false
if !thisLeads(t.Context(), client) {
t.Error("the leader took itself for a standby")
}
}