diff --git a/CHANGELOG.md b/CHANGELOG.md index 7663057..c1749fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,10 +20,21 @@ is only released once its **exit criterion** is met, not when its code lands. Until v1.0.0 the minor version is the breaking one: configuration formats and Go APIs change between minors without a deprecation period. -## Unreleased +## v0.10.5 — 2026-09-30 -Changes since v0.10.4, published as release candidates (v0.10.5-rc.N) until -there are enough for v0.10.5. +Published first as release candidates v0.10.5-rc.1 to rc.14; rc.5 failed its +format check and has no binaries. + +After upgrading: + +- **Declare what CPU models need.** A CPU model added from now on must say its + memory; one stored before this still loads and is flagged in `ctl models` + until `llmgw ctl model resources MODEL --memory GB --cores N` is run. Cores + are then reserved per replica and become the engine's thread count, for + replicas started after the change: `llmgw ctl restart --model MODEL`. +- **Upgrade the mirror on its own host**, now with one command: + `sudo llmgw upgrade v0.10.5 --service llmgw-mirror` (by hand once, for a + mirror older than this). `llmgw ctl upgrade` says so at the end. ### CPU cores are reserved, and memory has to be declared @@ -132,6 +143,13 @@ decisions rather than only being printed: A model never measured behaves exactly as before. Measurements are read from the store every half minute. +### Upgrading a machine the control plane does not reach + +`sudo llmgw upgrade VERSION --service UNIT` upgrades the machine it runs on — +the mirror's host above all, whose service runs unprivileged and cannot replace +its own binary as an agent does. The release's checksums are checked, the old +binary kept, the unit restarted and watched until it stays up. + ### Nodes report their own heat and power The first step of facility-aware placement (docs/facility-aware-placement.md), diff --git a/docs/install.md b/docs/install.md index 43b0845..1d11953 100644 --- a/docs/install.md +++ b/docs/install.md @@ -940,8 +940,18 @@ restart either — it rebuilds the fleet from what the agents report. fields, and both sides treat an absent one as unknown rather than as a default. Agents older than v0.10.4 cannot upgrade themselves: move those by hand once, installing the release binary and restarting `llmgw-agent`, and every upgrade -after that is the command above. The mirror, which is not an agent, is -upgraded the same way by hand. +after that is the command above. + +**The mirror is upgraded on its own host**, since it is not an agent and its +service cannot replace its own binary: + +``` +sudo llmgw upgrade v0.10.5 --service llmgw-mirror +``` + +The same checks — the release's checksums, the old binary kept beside the new — +then a restart of the unit, waiting for it to stay up. A mirror older than +v0.10.5 does not have the command: install the release binary by hand once. Then confirm what you have, rather than assuming: diff --git a/internal/cli/ctl_upgrade.go b/internal/cli/ctl_upgrade.go index 2a57915..150dff6 100644 --- a/internal/cli/ctl_upgrade.go +++ b/internal/cli/ctl_upgrade.go @@ -172,6 +172,10 @@ func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient, } } fmt.Fprintf(out, "%d node(s) upgraded to %s, %d already on it\n", done, v, skipped) + // The one machine this does not reach. Said every time, because it is + // exactly the kind of thing that stays three versions behind unnoticed. + fmt.Fprintf(out, "The model mirror's host is not upgraded by this; on it, run:\n"+ + " sudo llmgw upgrade %s --service llmgw-mirror\n", v) return nil } diff --git a/internal/cli/root.go b/internal/cli/root.go index 0ed9c07..573733e 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -47,6 +47,7 @@ func Root() *cobra.Command { newSimEngineCmd(), newDoctorCmd(), newNodeCmd(), + newSelfUpgradeCmd(), ) return root } diff --git a/internal/cli/selfupgrade.go b/internal/cli/selfupgrade.go new file mode 100644 index 0000000..49e7995 --- /dev/null +++ b/internal/cli/selfupgrade.go @@ -0,0 +1,87 @@ +package cli + +import ( + "errors" + "fmt" + "os" + "os/exec" + "runtime" + "strings" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/release" + "gitea.mynodes.xyz/ilker/llmgw/internal/version" +) + +// newSelfUpgradeCmd upgrades this machine's llmgw on its own, for the machines +// `llmgw ctl upgrade` does not reach: a mirror host, above all, whose service +// runs unprivileged with the filesystem read-only and so cannot replace its own +// binary the way an agent does. +func newSelfUpgradeCmd() *cobra.Command { + var releases, service string + c := &cobra.Command{ + Use: "upgrade VERSION [--service UNIT]", + Short: "Upgrade this machine's llmgw, and restart the service that runs it", + Long: "For a machine the control plane does not upgrade — the model mirror's host.\n" + + "The control plane and the nodes are upgraded with `llmgw ctl upgrade`.\n\n" + + "Fetches the release for this platform, checks it against the release's\n" + + "checksums, keeps the old binary beside the new one, and restarts --service,\n" + + "waiting for it to be running again. Run it with sudo.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + v := args[0] + out := cmd.OutOrStdout() + if !release.Tag.MatchString(v) { + return fmt.Errorf("%q is not a release version (v1.2.3 or v1.2.3-rc.4)", v) + } + if version.Version == v { + fmt.Fprintf(out, "this machine already runs %s\n", v) + return nil + } + if os.Geteuid() != 0 { + return errors.New("replacing this machine's binary needs root: run it with sudo") + } + self, err := os.Executable() + if err != nil { + return err + } + backup, err := release.Install(cmd.Context(), nil, releases, v, runtime.GOOS, runtime.GOARCH, self) + if err != nil { + return err + } + fmt.Fprintf(out, "%s -> %s: installed, checksum verified; the old binary is %s\n", + version.Version, v, backup) + if service == "" { + return nil + } + //nolint:gosec // the unit name is the operator's own flag + if b, err := exec.CommandContext(cmd.Context(), "systemctl", "restart", service).CombinedOutput(); err != nil { + return fmt.Errorf("restarting %s: %v: %s", service, err, strings.TrimSpace(string(b))) + } + // Running, and still running a few seconds on: a unit that starts + // and dies on a bad flag is "active" for a moment first. + deadline := time.Now().Add(30 * time.Second) + steady := 0 + for time.Now().Before(deadline) { + //nolint:gosec // as above + b, _ := exec.CommandContext(cmd.Context(), "systemctl", "is-active", service).Output() + if strings.TrimSpace(string(b)) == "active" { + if steady++; steady >= 3 { + fmt.Fprintf(out, "%s restarted and running\n", service) + return nil + } + } else { + steady = 0 + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("%s is not running after the upgrade; see journalctl -u %s, and %s holds the previous binary", + service, service, backup) + }, + } + c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from") + c.Flags().StringVar(&service, "service", "", "the systemd unit to restart afterwards, e.g. llmgw-mirror") + return c +}