From c5e388807a5a5bb9522a8c1cac3de2fbe198b10a Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Mon, 5 Oct 2026 02:43:33 +0200 Subject: [PATCH] ctl upgrade in waves, and the other control plane first One node at a time was hours on a fleet of hundreds. Nodes now go one, then two, four, up to --parallel at once, each wave waited on whole; a wave that does not come back stops it. The other control plane is upgraded first over ssh, so a leader restarted last hands over to a standby on the release. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 33 +++++ docs/install.md | 36 +++--- internal/cli/ctl_upgrade.go | 216 +++++++++++++++++++++++-------- internal/cli/ctl_upgrade_test.go | 37 +++++- 4 files changed, 245 insertions(+), 77 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f56e9a9..0d7f555 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,39 @@ is only released once its **exit criterion** is met, not when its code lands. Until v1.0.0 the minor version is the breaking one: configuration formats and Go APIs change between minors without a deprecation period. +## v0.11.3-rc.1 — 2026-10-05 + +Two control planes, run in anger on the reference cluster. + +### Added + +- **The console sets up a second control plane**: Do → Add a control plane + asks for the machine and a floating address and runs `server add`. +- **Who leads, everywhere.** `ctl status` and the console's Overview have a + `CONTROL` line: which control plane leads, which stand by, one that does not + answer, and whether the floating address is on the leader. +- **A standby's console is the leader's.** Its admin API passes `/api/` to the + leader over the agent port, with the cluster's certificates both ways, so + `sudo llmgw` and `llmgw ctl` show the same cluster on either machine. Control + plane certificates are now issued for client use too; `server add` reissues + older ones. +- **`ctl upgrade` in waves.** Nodes are upgraded one, then two, four… up to + `--parallel` (20) at once, so hundreds take minutes; a wave that does not + come back stops it. The other control plane is upgraded first, over ssh. +- `server add` says when the mirror's `--api-allow` would refuse the new + control plane. + +### Fixed + +- **Failover took 40 seconds; it takes a few.** A stopping control plane held + every agent's stream for the thirty seconds of a graceful shutdown; it now + gives up its lease first and lets the agents go at once. +- A control plane that lost its lease while running kept its agents, leaving + the new leader with no nodes. It now lets them go. +- An agent's reconnect backoff carried over from one outage to the next, so + each failover made the next one slower; it starts again after a session that + worked. + ## v0.11.2 — 2026-10-05 ### Routing by what each vLLM replica's cache holds diff --git a/docs/install.md b/docs/install.md index 2c0c677..3916b4e 100644 --- a/docs/install.md +++ b/docs/install.md @@ -2249,27 +2249,33 @@ joined with `node add`. Why, and how, is in [kubernetes.md](kubernetes.md). ## Upgrading -On the control plane: +On a control plane: ``` -sudo llmgw ctl upgrade v0.10.4 +sudo --preserve-env=SSH_AUTH_SOCK llmgw ctl upgrade v0.10.4 ``` -It upgrades this machine first — fetching the release for its platform, -checking it against the release's checksums, keeping the old binary as -`/usr/local/bin/llmgw.bak-pre-`, restarting `llmgw-server` and waiting -for every node to reconnect — and then each node in turn. A node's agent is told -the version over the stream it already holds, fetches and checks the release -itself, swaps it in and restarts; no ssh into the nodes is needed. Release -candidates (`v0.10.4-rc.2`) work the same way. +With two control planes it upgrades the other one first, over ssh as the user +who ran sudo (`--ssh-user`), with `llmgw upgrade` there; then this machine — +fetching the release for its platform, checking it against the release's +checksums, keeping the old binary as `/usr/local/bin/llmgw.bak-pre-`, +restarting `llmgw-server` and waiting for every node to reconnect. A leader +restarted last hands over to a standby already on the release. + +Then the nodes, in waves: one first, then two, four, and so on up to +`--parallel` (20) at once. A node's agent is told the version over the stream +it already holds, fetches and checks the release itself, swaps it in and +restarts; no ssh into the nodes is needed. A fleet of hundreds takes minutes, +not hours. Release candidates (`v0.10.4-rc.2`) work the same way. A node counts as done when it is back on the new version, ready, with as many -ready replicas as before. One that is not within `--timeout` (three minutes) -stops the upgrade there, so a bad release reaches one node rather than all of -them; `llmgw ctl logs ` says why, and the node keeps its previous binary -beside the new one. Nodes already on the version are skipped, so running the -command again resumes where it stopped. `--nodes a,b` limits it to some nodes, -and `--skip-control-plane` leaves this machine alone. +ready replicas as before. A wave not back within `--timeout` (three minutes) +stops the upgrade there — the first wave is one node, so a bad release reaches +one node rather than all of them; `llmgw ctl logs ` says why, and the node +keeps its previous binary beside the new one. Nodes already on the version are +skipped, so running the command again resumes where it stopped. `--nodes a,b` +limits it to some nodes, `--skip-control-plane` leaves the control planes alone, +and `--other-control-planes=false` this machine's partner. **Nothing stops serving.** Engines are started with `KillMode=process`, so they keep running when the agent restarts, and the agent reports them back on its diff --git a/internal/cli/ctl_upgrade.go b/internal/cli/ctl_upgrade.go index c119e0c..c2cbce8 100644 --- a/internal/cli/ctl_upgrade.go +++ b/internal/cli/ctl_upgrade.go @@ -16,6 +16,7 @@ import ( "github.com/spf13/cobra" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/onboard" "gitea.mynodes.xyz/ilker/llmgw/internal/release" "gitea.mynodes.xyz/ilker/llmgw/internal/version" ) @@ -27,19 +28,25 @@ func newUpgradeCmd(client func() *ctlClient) *cobra.Command { releases string perNode time.Duration serverSvc string + parallel int + sshUser string + othersCP bool ) c := &cobra.Command{ Use: "upgrade VERSION", - Short: "Upgrade this control plane, then every node, one at a time", + Short: "Upgrade the control planes, then every node, in waves", Long: "Moves the cluster to a release: v0.10.4, or a candidate like v0.10.4-rc.2.\n\n" + - "This machine first — run it with sudo on the control plane, since it\n" + - "replaces /usr/local/bin/llmgw and restarts llmgw-server. Then each node, in\n" + - "turn: its agent fetches the release for its own platform, checks it\n" + - "against the release's checksums, swaps it in and restarts. Engines keep\n" + - "serving through all of it and are adopted again; no ssh into the nodes is\n" + - "needed.\n\n" + + "The other control planes first, over ssh as --ssh-user (the one sudo was\n" + + "run by), then this machine — run it with sudo on a control plane, since it\n" + + "replaces /usr/local/bin/llmgw and restarts llmgw-server. A leader restarted\n" + + "last hands over to a standby already on the release.\n\n" + + "Then the nodes, in waves: one first, then twice as many each time a wave\n" + + "comes back whole, up to --parallel at once. Each agent fetches the release\n" + + "for its own platform, checks it against the release's checksums, swaps it\n" + + "in and restarts; engines keep serving through all of it and are adopted\n" + + "again, and no ssh into the nodes is needed.\n\n" + "A node counts as done when it is back on the new version, ready, with as\n" + - "many ready replicas as before. One that is not within --timeout stops the\n" + + "many ready replicas as before. A wave not back within --timeout stops the\n" + "upgrade there, so a bad release reaches one node rather than all of them.\n" + "Nodes already on the version are skipped, so running it again resumes.", Args: cobra.ExactArgs(1), @@ -61,11 +68,16 @@ func newUpgradeCmd(client func() *ctlClient) *cobra.Command { } } if !skipCP { + if othersCP { + if err := upgradeOtherControlPlanes(ctx, out, client, v, releases, sshUser); err != nil { + return err + } + } if err := upgradeControlPlane(ctx, out, client, v, releases, serverSvc); err != nil { return err } } - return upgradeNodes(ctx, out, client, v, nodes, perNode) + return upgradeNodes(ctx, out, client, v, nodes, perNode, parallel) }, } c.Flags().StringSliceVar(&nodes, "nodes", nil, "only these nodes (default: all)") @@ -74,9 +86,47 @@ func newUpgradeCmd(client func() *ctlClient) *cobra.Command { "(default: as server.yaml says, the mirror's copy on an offline cluster, else the project's)") c.Flags().DurationVar(&perNode, "timeout", 3*time.Minute, "how long a node may take to come back") c.Flags().StringVar(&serverSvc, "service", "llmgw-server", "this machine's systemd unit") + c.Flags().IntVar(¶llel, "parallel", 20, "the most nodes upgraded at once, once the first waves are back") + c.Flags().StringVar(&sshUser, "ssh-user", os.Getenv("SUDO_USER"), "who to log in to the other control planes as") + c.Flags().BoolVar(&othersCP, "other-control-planes", true, "upgrade the cluster's other control planes too, over ssh") return c } +// upgradeOtherControlPlanes moves every other control plane of the cluster to +// v, over ssh with `llmgw upgrade`, before this machine: each is a standby, or +// becomes one when this one is restarted, and the leadership should move to +// a machine already on the release. Nothing is changed here if one cannot be +// reached. +func upgradeOtherControlPlanes(ctx context.Context, out io.Writer, client func() *ctlClient, v, releases, user string) error { + var sum adminapi.Summary + if err := client().get(ctx, "/api/v1/summary", &sum); err != nil { + return err + } + for _, cp := range sum.ControlPlanes.Members { + if cp.This { + continue + } + r := &onboard.SSH{Addr: cp.Address, User: user} + have, err := r.Run(ctx, "llmgw --version", nil) + if err != nil { + return fmt.Errorf("%s (%s) cannot be reached over ssh as %q: %v\n"+ + "run `sudo llmgw upgrade %s --service llmgw-server` there, then this again with "+ + "--other-control-planes=false", cp.Name, cp.Address, user, err, v) + } + if strings.Contains(string(have), " "+v+"\n") || strings.HasSuffix(strings.TrimSpace(string(have)), " "+v) { + fmt.Fprintf(out, "== %s already runs %s\n", cp.Name, v) + continue + } + fmt.Fprintf(out, "== %s (%s): %s -> %s\n", cp.Name, cp.State, strings.TrimSpace(string(have)), v) + res, err := r.Run(ctx, "sudo llmgw upgrade "+shellQuote(v)+" --service llmgw-server --releases "+shellQuote(releases), nil) + if err != nil { + return fmt.Errorf("upgrading %s: %v: %s", cp.Name, err, strings.TrimSpace(string(res))) + } + fmt.Fprintf(out, " upgraded and restarted\n") + } + return nil +} + func upgradeControlPlane(ctx context.Context, out io.Writer, client func() *ctlClient, v, releases, svc string) error { if version.Version == v { fmt.Fprintf(out, "this machine already runs %s\n", v) @@ -133,13 +183,23 @@ func upgradeControlPlane(ctx context.Context, out io.Writer, client func() *ctlC } } -func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient, v string, only []string, perNode time.Duration) error { +// upgradeNodes moves nodes to v in waves: one node first, then twice as +// many each time one comes back whole, up to parallel at once. +// +// Waves rather than one at a time, because one at a time is hours on a +// fleet of hundreds, and an agent's restart costs nothing served — the +// engines keep running and are adopted again. One first, because a release +// that does not come back should reach one node rather than a wave of them. +func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient, v string, only []string, + perNode time.Duration, parallel int, +) error { var nodes []adminapi.NodeView if err := client().get(ctx, "/api/v1/nodes", &nodes); err != nil { return err } slices.SortFunc(nodes, func(a, b adminapi.NodeView) int { return strings.Compare(a.ID, b.ID) }) - done, skipped := 0, 0 + var todo []adminapi.NodeView + skipped := 0 for _, n := range nodes { if len(only) > 0 && !slices.Contains(only, n.ID) { continue @@ -148,41 +208,21 @@ func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient, skipped++ continue } - start := time.Now() - before := readyReplicasOn(ctx, client, n.ID) - fmt.Fprintf(out, "== %s: %s -> %s\n", n.ID, n.AgentVersion, v) - body, _ := json.Marshal(adminapi.UpgradeRequest{Version: v}) - if err := client().post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil { - return fmt.Errorf("%s: %w\nstopped here; `llmgw ctl logs %s` says more", n.ID, err, n.ID) - } - fmt.Fprintf(out, " new binary in place; waiting for it to come back with its %d replica(s)\n", before) - deadline := time.Now().Add(perNode) - for { - var now []adminapi.NodeView - _ = client().get(ctx, "/api/v1/nodes", &now) - var cur adminapi.NodeView - for _, x := range now { - if x.ID == n.ID { - cur = x - } - } - after := readyReplicasOn(ctx, client, n.ID) - if cur.AgentVersion == v && cur.State == "ready" && after >= before { - fmt.Fprintf(out, " done in %s: ready, replicas %d/%d\n", - time.Since(start).Round(time.Second), after, before) - done++ - break - } - if time.Now().After(deadline) { - return fmt.Errorf("%s did not come back within %s (version %q, state %q, replicas %d/%d)\n"+ - "stopped here, before the next node; `llmgw ctl logs %s` says why, and the node "+ - "keeps its previous binary beside the new one", n.ID, perNode, cur.AgentVersion, - cur.State, after, before, n.ID) - } - time.Sleep(2 * time.Second) - } + todo = append(todo, n) } - fmt.Fprintf(out, "%d node(s) upgraded to %s, %d already on it\n", done, v, skipped) + parallel = max(parallel, 1) + start, size, done := time.Now(), 1, 0 + for len(todo) > 0 { + wave := todo[:min(size, len(todo))] + todo = todo[len(wave):] + if err := upgradeWave(ctx, out, client, v, wave, perNode); err != nil { + return fmt.Errorf("%w\nstopped after this wave: %d upgraded, %d not tried", err, done, len(todo)) + } + done += len(wave) + size = min(size*2, parallel) + } + fmt.Fprintf(out, "%d node(s) upgraded to %s in %s, %d already on it\n", + done, v, time.Since(start).Round(time.Second), skipped) // The one machine this does not reach. Said every time, because it is // exactly the kind of thing that stays three versions behind unnoticed. fmt.Fprintf(out, "The model mirror's host is not upgraded by this; on it, run:\n"+ @@ -190,23 +230,87 @@ func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient, return nil } -// readyReplicasOn counts the ready replicas with a shard on a node. -func readyReplicasOn(ctx context.Context, client func() *ctlClient, node string) int { - var reps []adminapi.ReplicaView - if err := client().get(ctx, "/api/v1/replicas", &reps); err != nil { - return 0 +// upgradeWave tells every node in the wave at once, and waits until each is +// back on v, ready, with as many ready replicas as it had. +func upgradeWave(ctx context.Context, out io.Writer, client func() *ctlClient, v string, + wave []adminapi.NodeView, perNode time.Duration, +) error { + start := time.Now() + before := readyReplicasByNode(ctx, client) + ids := make([]string, len(wave)) + for i, n := range wave { + ids[i] = n.ID + } + fmt.Fprintf(out, "== %d node(s): %s\n", len(wave), abbreviate(ids, 8)) + body, _ := json.Marshal(adminapi.UpgradeRequest{Version: v}) + var failed []string + for _, n := range wave { + if err := client().post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil { + failed = append(failed, fmt.Sprintf("%s: %v", n.ID, err)) + } + } + if len(failed) > 0 { + return fmt.Errorf("not told to upgrade:\n %s\n`llmgw ctl logs NODE` says more", strings.Join(failed, "\n ")) + } + waiting := slices.Clone(ids) + deadline := time.Now().Add(perNode) + for { + var now []adminapi.NodeView + _ = client().get(ctx, "/api/v1/nodes", &now) + after := readyReplicasByNode(ctx, client) + state := map[string]adminapi.NodeView{} + for _, x := range now { + state[x.ID] = x + } + waiting = slices.DeleteFunc(waiting, func(id string) bool { + cur := state[id] + return cur.AgentVersion == v && cur.State == "ready" && after[id] >= before[id] + }) + if len(waiting) == 0 { + fmt.Fprintf(out, " done in %s\n", time.Since(start).Round(time.Second)) + return nil + } + if time.Now().After(deadline) { + var why []string + for _, id := range waiting { + cur := state[id] + why = append(why, fmt.Sprintf("%s (version %q, state %q, replicas %d/%d)", + id, cur.AgentVersion, cur.State, after[id], before[id])) + } + return fmt.Errorf("not back within %s:\n %s\n`llmgw ctl logs NODE` says why, and each keeps its "+ + "previous binary beside the new one", perNode, strings.Join(why, "\n ")) + } + time.Sleep(2 * time.Second) + } +} + +// abbreviate lists up to n names, and says how many more. +func abbreviate(names []string, n int) string { + if len(names) <= n { + return strings.Join(names, ", ") + } + return strings.Join(names[:n], ", ") + fmt.Sprintf(" and %d more", len(names)-n) +} + +// readyReplicasByNode counts, for each node, the ready replicas with a +// shard on it. +func readyReplicasByNode(ctx context.Context, client func() *ctlClient) map[string]int { + var reps []adminapi.ReplicaView + out := map[string]int{} + if err := client().get(ctx, "/api/v1/replicas", &reps); err != nil { + return out } - n := 0 for _, r := range reps { if r.State != "ready" { continue } - for _, s := range r.Shards { - if s.Node == node { - n++ - break + seen := map[string]bool{} + for _, sh := range r.Shards { + if !seen[sh.Node] { + seen[sh.Node] = true + out[sh.Node]++ } } } - return n + return out } diff --git a/internal/cli/ctl_upgrade_test.go b/internal/cli/ctl_upgrade_test.go index 1a8636e..94d7bc2 100644 --- a/internal/cli/ctl_upgrade_test.go +++ b/internal/cli/ctl_upgrade_test.go @@ -3,6 +3,7 @@ package cli import ( "bytes" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" @@ -57,32 +58,56 @@ func (f *fakeFleet) server(t *testing.T) *httptest.Server { return srv } -func TestAnUpgradeGoesNodeByNodeAndSkipsTheDone(t *testing.T) { +func TestAnUpgradeSkipsTheNodesAlreadyDone(t *testing.T) { f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.1", "c": "v1.0.0"}} srv := f.server(t) var out bytes.Buffer - err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 5*time.Second) + err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 5*time.Second, 1) if err != nil { t.Fatal(err) } if strings.Join(f.told, ",") != "a,c" { t.Errorf("told %v; b was already on the version", f.told) } - if !strings.Contains(out.String(), "2 node(s) upgraded to v1.0.1, 1 already on it") { + if !strings.Contains(out.String(), "2 node(s) upgraded to v1.0.1 in") || !strings.Contains(out.String(), "1 already on it") { t.Errorf("output:\n%s", out.String()) } } -func TestAnUpgradeStopsAtTheFirstNodeThatDoesNotComeBack(t *testing.T) { +func TestAnUpgradeStopsAtTheFirstWaveThatDoesNotComeBack(t *testing.T) { // A bad release reaches one node, not all of them. f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.0"}, stuck: map[string]bool{"a": true}} srv := f.server(t) var out bytes.Buffer - err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 3*time.Second) - if err == nil || !strings.Contains(err.Error(), "a did not come back") { + err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 3*time.Second, 1) + if err == nil || !strings.Contains(err.Error(), "not back within") || !strings.Contains(err.Error(), "a (version") { t.Fatalf("err %v", err) } if len(f.told) != 1 { t.Errorf("went on to %v after a node failed to come back", f.told) } } + +func TestAnUpgradeGoesInWavesThatGrowUpToTheLimit(t *testing.T) { + // One first, then twice as many each time, up to --parallel: hundreds of + // nodes in minutes rather than hours, and a bad release still on one. + versions := map[string]string{} + for i := range 12 { + versions[fmt.Sprintf("n%02d", i)] = "v1.0.0" + } + f := &fakeFleet{versions: versions} + srv := f.server(t) + var out bytes.Buffer + if err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 5*time.Second, 4); err != nil { + t.Fatal(err) + } + var sizes []string + for _, line := range strings.Split(out.String(), "\n") { + if n, ok := strings.CutPrefix(line, "== "); ok { + sizes = append(sizes, strings.Fields(n)[0]) + } + } + if got := strings.Join(sizes, ","); got != "1,2,4,4,1" { + t.Errorf("waves %s:\n%s", got, out.String()) + } +}