package cli import ( "context" "errors" "fmt" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "io" "os" "os/exec" "runtime" "strings" "time" "github.com/spf13/cobra" "gitea.mynodes.xyz/ilker/llmgw/internal/release" "gitea.mynodes.xyz/ilker/llmgw/internal/version" ) // newSelfUpgradeCmd upgrades this machine's llmgw on its own, for the machines // `llmgw ctl upgrade` does not reach: a mirror host, above all, whose service // runs unprivileged with the filesystem read-only and so cannot replace its own // binary the way an agent does. func newSelfUpgradeCmd() *cobra.Command { var releases, service string c := &cobra.Command{ Use: "upgrade VERSION [--service UNIT]", Short: "Upgrade this machine's llmgw, and restart the service that runs it", Long: "For a machine the control plane does not upgrade — the model mirror's host.\n" + "The control plane and the nodes are upgraded with `llmgw ctl upgrade`.\n\n" + "Fetches the release for this platform, checks it against the release's\n" + "checksums, keeps the old binary beside the new one, and restarts --service,\n" + "waiting for it to be running again. Run it with sudo.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { v := args[0] out := cmd.OutOrStdout() if !release.Tag.MatchString(v) { return fmt.Errorf("%q is not a release version (v1.2.3 or v1.2.3-rc.4)", v) } if version.Version == v { fmt.Fprintf(out, "this machine already runs %s\n", v) return nil } if os.Geteuid() != 0 { return errors.New("replacing this machine's binary needs root: run it with sudo") } self, err := os.Executable() if err != nil { return err } backup, err := release.Install(cmd.Context(), nil, releases, v, runtime.GOOS, runtime.GOARCH, self) if err != nil { return err } fmt.Fprintf(out, "%s -> %s: installed, checksum verified; the old binary is %s\n", version.Version, v, backup) if service == "" { return nil } //nolint:gosec // the unit name is the operator's own flag if b, err := exec.CommandContext(cmd.Context(), "systemctl", "restart", service).CombinedOutput(); err != nil { return fmt.Errorf("restarting %s: %v: %s", service, err, strings.TrimSpace(string(b))) } // Running, and still running a few seconds on: a unit that starts // and dies on a bad flag is "active" for a moment first. deadline := time.Now().Add(30 * time.Second) steady := 0 for time.Now().Before(deadline) { //nolint:gosec // as above b, _ := exec.CommandContext(cmd.Context(), "systemctl", "is-active", service).Output() if strings.TrimSpace(string(b)) == "active" { if steady++; steady >= 3 { fmt.Fprintf(out, "%s restarted and running\n", service) if service == "llmgw-server" { nodesLeftBehind(cmd.Context(), out, newCtlClient("127.0.0.1:9090"), v) } return nil } } else { steady = 0 } time.Sleep(2 * time.Second) } return fmt.Errorf("%s is not running after the upgrade; see journalctl -u %s, and %s holds the previous binary", service, service, backup) }, } c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from") c.Flags().StringVar(&service, "service", "", "the systemd unit to restart afterwards, e.g. llmgw-mirror") return c } // nodesLeftBehind says, after a control plane alone was upgraded, that its // nodes were not, and how they are: this command looks like the cluster's // upgrade and is not, and a site's nodes stayed a release behind after it. func nodesLeftBehind(ctx context.Context, out io.Writer, c *ctlClient, v string) { var nodes []adminapi.NodeView _ = c.patient(20*time.Second).get(ctx, "/api/v1/nodes", &nodes) behind := 0 for _, n := range nodes { if n.AgentVersion != v { behind++ } } switch { case len(nodes) == 0: fmt.Fprintf(out, "This upgraded this machine only. The nodes are upgraded from the leading "+ "control plane with:\n sudo llmgw ctl upgrade %s --skip-control-plane\n", v) case behind > 0: fmt.Fprintf(out, "This upgraded this machine only: %d of %d node(s) are not on %s. On the leading "+ "control plane, run:\n sudo llmgw ctl upgrade %s --skip-control-plane\n", behind, len(nodes), v, v) } }