ctl upgrade upgrades the leader last, wherever it is run
Run on the standby, it upgraded the leader first, then itself, and the leadership moved twice. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
abaf086fd5
commit
ef33e52d3d
4 files changed
+60
-12
No files matched your search
@@ -55,6 +55,8 @@ Two control planes, run in anger on the reference cluster.
|
||||
|
||||
### Fixed
|
||||
|
||||
- `ctl upgrade` upgrades the leader last wherever it is run; run on the
|
||||
standby, it upgraded the leader first and the leadership moved twice.
|
||||
- **Failover took 40 seconds; it takes a few.** A stopping control plane held
|
||||
every agent's stream for the thirty seconds of a graceful shutdown; it now
|
||||
gives up its lease first and lets the agents go at once.
|
||||
|
||||
+5
-4
@@ -2259,12 +2259,13 @@ On a control plane:
|
||||
sudo --preserve-env=SSH_AUTH_SOCK llmgw ctl upgrade v0.10.4
|
||||
```
|
||||
|
||||
With two control planes it upgrades the other one first, over ssh as the user
|
||||
who ran sudo (`--ssh-user`), with `llmgw upgrade` there; then this machine —
|
||||
With two control planes it upgrades the standby first and the leader last,
|
||||
whichever this machine is — the other over ssh as the user who ran sudo
|
||||
(`--ssh-user`), with `llmgw upgrade` there, and this machine —
|
||||
fetching the release for its platform, checking it against the release's
|
||||
checksums, keeping the old binary as `/usr/local/bin/llmgw.bak-pre-<version>`,
|
||||
restarting `llmgw-server` and waiting for every node to reconnect. A leader
|
||||
restarted last hands over to a standby already on the release.
|
||||
restarting `llmgw-server` and waiting for every node to reconnect. The
|
||||
leader's restart hands over once, to a standby already on the release.
|
||||
|
||||
Then the nodes, in waves: one first, then two, four, and so on up to
|
||||
`--parallel` (20) at once. A node's agent is told the version over the stream
|
||||
|
||||
@@ -68,14 +68,26 @@ func newUpgradeCmd(client func() *ctlClient) *cobra.Command {
|
||||
}
|
||||
}
|
||||
if !skipCP {
|
||||
if othersCP {
|
||||
if err := upgradeOtherControlPlanes(ctx, out, client, v, releases, sshUser); err != nil {
|
||||
// The leader last, wherever this runs: every other control
|
||||
// plane is a standby, upgraded with nothing moving, and the
|
||||
// leader's restart hands over once, to a machine already on
|
||||
// the release.
|
||||
others := func() error {
|
||||
if !othersCP {
|
||||
return nil
|
||||
}
|
||||
return upgradeOtherControlPlanes(ctx, out, client, v, releases, sshUser)
|
||||
}
|
||||
self := func() error { return upgradeControlPlane(ctx, out, client, v, releases, serverSvc) }
|
||||
order := []func() error{others, self}
|
||||
if !thisLeads(ctx, client) {
|
||||
order = []func() error{self, others}
|
||||
}
|
||||
for _, step := range order {
|
||||
if err := step(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := upgradeControlPlane(ctx, out, client, v, releases, serverSvc); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return upgradeNodes(ctx, out, client, v, nodes, perNode, parallel)
|
||||
},
|
||||
@@ -92,10 +104,23 @@ func newUpgradeCmd(client func() *ctlClient) *cobra.Command {
|
||||
return c
|
||||
}
|
||||
|
||||
// thisLeads says whether this machine's control plane leads; one that does
|
||||
// not say, alone or older, is taken to.
|
||||
func thisLeads(ctx context.Context, client func() *ctlClient) bool {
|
||||
var sum adminapi.Summary
|
||||
if err := client().get(ctx, "/api/v1/summary", &sum); err != nil {
|
||||
return true
|
||||
}
|
||||
for _, cp := range sum.ControlPlanes.Members {
|
||||
if cp.This {
|
||||
return cp.State == "leading"
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// upgradeOtherControlPlanes moves every other control plane of the cluster to
|
||||
// v, over ssh with `llmgw upgrade`, before this machine: each is a standby, or
|
||||
// becomes one when this one is restarted, and the leadership should move to
|
||||
// a machine already on the release. Nothing is changed here if one cannot be
|
||||
// v, over ssh with `llmgw upgrade`. Nothing is changed here if one cannot be
|
||||
// reached.
|
||||
func upgradeOtherControlPlanes(ctx context.Context, out io.Writer, client func() *ctlClient, v, releases, user string) error {
|
||||
var sum adminapi.Summary
|
||||
|
||||
@@ -111,3 +111,23 @@ func TestAnUpgradeGoesInWavesThatGrowUpToTheLimit(t *testing.T) {
|
||||
t.Errorf("waves %s:\n%s", got, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpgradeKnowsWhetherItRunsOnTheLeader(t *testing.T) {
|
||||
sum := adminapi.Summary{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(sum)
|
||||
}))
|
||||
defer srv.Close()
|
||||
client := func() *ctlClient { return newCtlClient(srv.URL) }
|
||||
if !thisLeads(t.Context(), client) {
|
||||
t.Error("a control plane with no rival does not lead")
|
||||
}
|
||||
sum.ControlPlanes.Members = []adminapi.ControlPlane{{Name: "a", State: "leading"}, {Name: "b", State: "standing by", This: true}}
|
||||
if thisLeads(t.Context(), client) {
|
||||
t.Error("a standby took itself for the leader: it would upgrade the leader before itself")
|
||||
}
|
||||
sum.ControlPlanes.Members[0].This, sum.ControlPlanes.Members[1].This = true, false
|
||||
if !thisLeads(t.Context(), client) {
|
||||
t.Error("the leader took itself for a standby")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user