llmgw/internal/cli/ctl_upgrade.go

package cli

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"runtime"
	"slices"
	"strings"
	"time"

	"github.com/spf13/cobra"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
	"gitea.mynodes.xyz/ilker/llmgw/internal/release"
	"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)

func newUpgradeCmd(client func() *ctlClient) *cobra.Command {
	var (
		nodes     []string
		skipCP    bool
		releases  string
		perNode   time.Duration
		serverSvc string
	)
	c := &cobra.Command{
		Use:   "upgrade VERSION",
		Short: "Upgrade this control plane, then every node, one at a time",
		Long: "Moves the cluster to a release: v0.10.4, or a candidate like v0.10.4-rc.2.\n\n" +
			"This machine first — run it with sudo on the control plane, since it\n" +
			"replaces /usr/local/bin/llmgw and restarts llmgw-server. Then each node, in\n" +
			"turn: its agent fetches the release for its own platform, checks it\n" +
			"against the release's checksums, swaps it in and restarts. Engines keep\n" +
			"serving through all of it and are adopted again; no ssh into the nodes is\n" +
			"needed.\n\n" +
			"A node counts as done when it is back on the new version, ready, with as\n" +
			"many ready replicas as before. One that is not within --timeout stops the\n" +
			"upgrade there, so a bad release reaches one node rather than all of them.\n" +
			"Nodes already on the version are skipped, so running it again resumes.",
		Args: cobra.ExactArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			v := args[0]
			if !release.Tag.MatchString(v) {
				return fmt.Errorf("%q is not a release version (v1.2.3 or v1.2.3-rc.4)", v)
			}
			out := cmd.OutOrStdout()
			ctx := cmd.Context()
			if releases == "" {
				// Where the control plane sends its nodes: an offline
				// cluster's mirror, so this machine needs no internet either.
				if cfg, ok := loadServerConfig("/etc/llmgw/server.yaml"); ok {
					releases = cfg.ReleasesBase()
				}
				if releases == "" {
					releases = release.DefaultBase
				}
			}
			if !skipCP {
				if err := upgradeControlPlane(ctx, out, client, v, releases, serverSvc); err != nil {
					return err
				}
			}
			return upgradeNodes(ctx, out, client, v, nodes, perNode)
		},
	}
	c.Flags().StringSliceVar(&nodes, "nodes", nil, "only these nodes (default: all)")
	c.Flags().BoolVar(&skipCP, "skip-control-plane", false, "leave this machine as it is and upgrade only nodes")
	c.Flags().StringVar(&releases, "releases", "", "where release binaries are fetched from, for this machine "+
		"(default: as server.yaml says, the mirror's copy on an offline cluster, else the project's)")
	c.Flags().DurationVar(&perNode, "timeout", 3*time.Minute, "how long a node may take to come back")
	c.Flags().StringVar(&serverSvc, "service", "llmgw-server", "this machine's systemd unit")
	return c
}

func upgradeControlPlane(ctx context.Context, out io.Writer, client func() *ctlClient, v, releases, svc string) error {
	if version.Version == v {
		fmt.Fprintf(out, "this machine already runs %s\n", v)
		return nil
	}
	if os.Geteuid() != 0 {
		return errors.New("replacing this machine's binary needs root: run it with sudo, " +
			"or pass --skip-control-plane to upgrade only the nodes")
	}
	self, err := os.Executable()
	if err != nil {
		return err
	}
	fmt.Fprintf(out, "== this machine: %s -> %s\n", version.Version, v)
	backup, err := release.Install(ctx, nil, releases, v, runtime.GOOS, runtime.GOARCH, self)
	if err != nil {
		return err
	}
	fmt.Fprintf(out, "   installed, checksum verified; the old binary is %s\n", backup)
	var before []adminapi.NodeView
	_ = client().get(ctx, "/api/v1/nodes", &before)
	if b, err := exec.CommandContext(ctx, "systemctl", "restart", svc).CombinedOutput(); err != nil { //nolint:gosec // fixed command
		return fmt.Errorf("restarting %s: %v: %s", svc, err, strings.TrimSpace(string(b)))
	}
	fmt.Fprintf(out, "   %s restarted; waiting for its nodes to reconnect\n", svc)
	// Back when the admin API answers and every node that was ready is ready
	// again: agents reconnect within seconds of the control plane returning.
	want := 0
	for _, n := range before {
		if n.State == "ready" {
			want++
		}
	}
	deadline := time.Now().Add(2 * time.Minute)
	for {
		var now []adminapi.NodeView
		if err := client().get(ctx, "/api/v1/nodes", &now); err == nil {
			ready := 0
			for _, n := range now {
				if n.State == "ready" {
					ready++
				}
			}
			if ready >= want {
				fmt.Fprintf(out, "   back: %d of %d nodes ready\n", ready, len(now))
				return nil
			}
		}
		if time.Now().After(deadline) {
			return fmt.Errorf("%s did not come back with its %d nodes within two minutes; "+
				"see journalctl -u %s, and %s holds the previous binary", svc, want, svc, backup)
		}
		time.Sleep(2 * time.Second)
	}
}

func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient, v string, only []string, perNode time.Duration) error {
	var nodes []adminapi.NodeView
	if err := client().get(ctx, "/api/v1/nodes", &nodes); err != nil {
		return err
	}
	slices.SortFunc(nodes, func(a, b adminapi.NodeView) int { return strings.Compare(a.ID, b.ID) })
	done, skipped := 0, 0
	for _, n := range nodes {
		if len(only) > 0 && !slices.Contains(only, n.ID) {
			continue
		}
		if n.AgentVersion == v {
			skipped++
			continue
		}
		start := time.Now()
		before := readyReplicasOn(ctx, client, n.ID)
		fmt.Fprintf(out, "== %s: %s -> %s\n", n.ID, n.AgentVersion, v)
		body, _ := json.Marshal(adminapi.UpgradeRequest{Version: v})
		if err := client().post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil {
			return fmt.Errorf("%s: %w\nstopped here; `llmgw ctl logs %s` says more", n.ID, err, n.ID)
		}
		fmt.Fprintf(out, "   new binary in place; waiting for it to come back with its %d replica(s)\n", before)
		deadline := time.Now().Add(perNode)
		for {
			var now []adminapi.NodeView
			_ = client().get(ctx, "/api/v1/nodes", &now)
			var cur adminapi.NodeView
			for _, x := range now {
				if x.ID == n.ID {
					cur = x
				}
			}
			after := readyReplicasOn(ctx, client, n.ID)
			if cur.AgentVersion == v && cur.State == "ready" && after >= before {
				fmt.Fprintf(out, "   done in %s: ready, replicas %d/%d\n",
					time.Since(start).Round(time.Second), after, before)
				done++
				break
			}
			if time.Now().After(deadline) {
				return fmt.Errorf("%s did not come back within %s (version %q, state %q, replicas %d/%d)\n"+
					"stopped here, before the next node; `llmgw ctl logs %s` says why, and the node "+
					"keeps its previous binary beside the new one", n.ID, perNode, cur.AgentVersion,
					cur.State, after, before, n.ID)
			}
			time.Sleep(2 * time.Second)
		}
	}
	fmt.Fprintf(out, "%d node(s) upgraded to %s, %d already on it\n", done, v, skipped)
	// The one machine this does not reach. Said every time, because it is
	// exactly the kind of thing that stays three versions behind unnoticed.
	fmt.Fprintf(out, "The model mirror's host is not upgraded by this; on it, run:\n"+
		"  sudo llmgw upgrade %s --service llmgw-mirror\n", v)
	return nil
}

// readyReplicasOn counts the ready replicas with a shard on a node.
func readyReplicasOn(ctx context.Context, client func() *ctlClient, node string) int {
	var reps []adminapi.ReplicaView
	if err := client().get(ctx, "/api/v1/replicas", &reps); err != nil {
		return 0
	}
	n := 0
	for _, r := range reps {
		if r.State != "ready" {
			continue
		}
		for _, s := range r.Shards {
			if s.Node == node {
				n++
				break
			}
		}
	}
	return n
}