llmgw/internal/cli/selfupgrade.go

package cli

import (
	"context"
	"errors"
	"fmt"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
	"io"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"time"

	"github.com/spf13/cobra"

	"gitea.mynodes.xyz/ilker/llmgw/internal/release"
	"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)

// newSelfUpgradeCmd upgrades this machine's llmgw on its own, for the machines
// `llmgw ctl upgrade` does not reach: a mirror host, above all, whose service
// runs unprivileged with the filesystem read-only and so cannot replace its own
// binary the way an agent does.
func newSelfUpgradeCmd() *cobra.Command {
	var releases, service string
	c := &cobra.Command{
		Use:   "upgrade VERSION [--service UNIT]",
		Short: "Upgrade this machine's llmgw, and restart the service that runs it",
		Long: "For a machine the control plane does not upgrade — the model mirror's host.\n" +
			"The control plane and the nodes are upgraded with `llmgw ctl upgrade`.\n\n" +
			"Fetches the release for this platform, checks it against the release's\n" +
			"checksums, keeps the old binary beside the new one, and restarts --service,\n" +
			"waiting for it to be running again. Run it with sudo.",
		Args: cobra.ExactArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			v := args[0]
			out := cmd.OutOrStdout()
			if !release.Tag.MatchString(v) {
				return fmt.Errorf("%q is not a release version (v1.2.3 or v1.2.3-rc.4)", v)
			}
			if version.Version == v {
				fmt.Fprintf(out, "this machine already runs %s\n", v)
				return nil
			}
			if os.Geteuid() != 0 {
				return errors.New("replacing this machine's binary needs root: run it with sudo")
			}
			self, err := os.Executable()
			if err != nil {
				return err
			}
			backup, err := release.Install(cmd.Context(), nil, releases, v, runtime.GOOS, runtime.GOARCH, self)
			if err != nil {
				return err
			}
			fmt.Fprintf(out, "%s -> %s: installed, checksum verified; the old binary is %s\n",
				version.Version, v, backup)
			if service == "" {
				return nil
			}
			//nolint:gosec // the unit name is the operator's own flag
			if b, err := exec.CommandContext(cmd.Context(), "systemctl", "restart", service).CombinedOutput(); err != nil {
				return fmt.Errorf("restarting %s: %v: %s", service, err, strings.TrimSpace(string(b)))
			}
			// Running, and still running a few seconds on: a unit that starts
			// and dies on a bad flag is "active" for a moment first.
			deadline := time.Now().Add(30 * time.Second)
			steady := 0
			for time.Now().Before(deadline) {
				//nolint:gosec // as above
				b, _ := exec.CommandContext(cmd.Context(), "systemctl", "is-active", service).Output()
				if strings.TrimSpace(string(b)) == "active" {
					if steady++; steady >= 3 {
						fmt.Fprintf(out, "%s restarted and running\n", service)
						if service == "llmgw-server" {
							nodesLeftBehind(cmd.Context(), out, newCtlClient("127.0.0.1:9090"), v)
						}
						return nil
					}
				} else {
					steady = 0
				}
				time.Sleep(2 * time.Second)
			}
			return fmt.Errorf("%s is not running after the upgrade; see journalctl -u %s, and %s holds the previous binary",
				service, service, backup)
		},
	}
	c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from")
	c.Flags().StringVar(&service, "service", "", "the systemd unit to restart afterwards, e.g. llmgw-mirror")
	return c
}

// nodesLeftBehind says, after a control plane alone was upgraded, that its
// nodes were not, and how they are: this command looks like the cluster's
// upgrade and is not, and a site's nodes stayed a release behind after it.
func nodesLeftBehind(ctx context.Context, out io.Writer, c *ctlClient, v string) {
	var nodes []adminapi.NodeView
	_ = c.patient(20*time.Second).get(ctx, "/api/v1/nodes", &nodes)
	behind := 0
	for _, n := range nodes {
		if n.AgentVersion != v {
			behind++
		}
	}
	switch {
	case len(nodes) == 0:
		fmt.Fprintf(out, "This upgraded this machine only. The nodes are upgraded from the leading "+
			"control plane with:\n  sudo llmgw ctl upgrade %s --skip-control-plane\n", v)
	case behind > 0:
		fmt.Fprintf(out, "This upgraded this machine only: %d of %d node(s) are not on %s. On the leading "+
			"control plane, run:\n  sudo llmgw ctl upgrade %s --skip-control-plane\n", behind, len(nodes), v, v)
	}
}