From 5cf0003e74c29663a529604ccbfef20af43008df Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 30 Sep 2026 22:26:19 +0200 Subject: [PATCH] Release v0.10.5 Collects rc.1 to rc.14: the mirror page in tabs with a card per model, one line per record in every admin table, reserved CPU cores and required memory, capacity classes and preemption, tenant capacity quotas, sizing by measured throughput, and node sensor readings. Adds `llmgw upgrade` for the mirror host, which `ctl upgrade` does not reach, and says so when it ends. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 24 ++++++++-- docs/install.md | 14 +++++- internal/cli/ctl_upgrade.go | 4 ++ internal/cli/root.go | 1 + internal/cli/selfupgrade.go | 87 +++++++++++++++++++++++++++++++++++++ 5 files changed, 125 insertions(+), 5 deletions(-) create mode 100644 internal/cli/selfupgrade.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 7663057..c1749fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,10 +20,21 @@ is only released once its **exit criterion** is met, not when its code lands. Until v1.0.0 the minor version is the breaking one: configuration formats and Go APIs change between minors without a deprecation period. -## Unreleased +## v0.10.5 — 2026-09-30 -Changes since v0.10.4, published as release candidates (v0.10.5-rc.N) until -there are enough for v0.10.5. +Published first as release candidates v0.10.5-rc.1 to rc.14; rc.5 failed its +format check and has no binaries. + +After upgrading: + +- **Declare what CPU models need.** A CPU model added from now on must say its + memory; one stored before this still loads and is flagged in `ctl models` + until `llmgw ctl model resources MODEL --memory GB --cores N` is run. Cores + are then reserved per replica and become the engine's thread count, for + replicas started after the change: `llmgw ctl restart --model MODEL`. +- **Upgrade the mirror on its own host**, now with one command: + `sudo llmgw upgrade v0.10.5 --service llmgw-mirror` (by hand once, for a + mirror older than this). `llmgw ctl upgrade` says so at the end. ### CPU cores are reserved, and memory has to be declared @@ -132,6 +143,13 @@ decisions rather than only being printed: A model never measured behaves exactly as before. Measurements are read from the store every half minute. +### Upgrading a machine the control plane does not reach + +`sudo llmgw upgrade VERSION --service UNIT` upgrades the machine it runs on — +the mirror's host above all, whose service runs unprivileged and cannot replace +its own binary as an agent does. The release's checksums are checked, the old +binary kept, the unit restarted and watched until it stays up. + ### Nodes report their own heat and power The first step of facility-aware placement (docs/facility-aware-placement.md), diff --git a/docs/install.md b/docs/install.md index 43b0845..1d11953 100644 --- a/docs/install.md +++ b/docs/install.md @@ -940,8 +940,18 @@ restart either — it rebuilds the fleet from what the agents report. fields, and both sides treat an absent one as unknown rather than as a default. Agents older than v0.10.4 cannot upgrade themselves: move those by hand once, installing the release binary and restarting `llmgw-agent`, and every upgrade -after that is the command above. The mirror, which is not an agent, is -upgraded the same way by hand. +after that is the command above. + +**The mirror is upgraded on its own host**, since it is not an agent and its +service cannot replace its own binary: + +``` +sudo llmgw upgrade v0.10.5 --service llmgw-mirror +``` + +The same checks — the release's checksums, the old binary kept beside the new — +then a restart of the unit, waiting for it to stay up. A mirror older than +v0.10.5 does not have the command: install the release binary by hand once. Then confirm what you have, rather than assuming: diff --git a/internal/cli/ctl_upgrade.go b/internal/cli/ctl_upgrade.go index 2a57915..150dff6 100644 --- a/internal/cli/ctl_upgrade.go +++ b/internal/cli/ctl_upgrade.go @@ -172,6 +172,10 @@ func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient, } } fmt.Fprintf(out, "%d node(s) upgraded to %s, %d already on it\n", done, v, skipped) + // The one machine this does not reach. Said every time, because it is + // exactly the kind of thing that stays three versions behind unnoticed. + fmt.Fprintf(out, "The model mirror's host is not upgraded by this; on it, run:\n"+ + " sudo llmgw upgrade %s --service llmgw-mirror\n", v) return nil } diff --git a/internal/cli/root.go b/internal/cli/root.go index 0ed9c07..573733e 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -47,6 +47,7 @@ func Root() *cobra.Command { newSimEngineCmd(), newDoctorCmd(), newNodeCmd(), + newSelfUpgradeCmd(), ) return root } diff --git a/internal/cli/selfupgrade.go b/internal/cli/selfupgrade.go new file mode 100644 index 0000000..49e7995 --- /dev/null +++ b/internal/cli/selfupgrade.go @@ -0,0 +1,87 @@ +package cli + +import ( + "errors" + "fmt" + "os" + "os/exec" + "runtime" + "strings" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/release" + "gitea.mynodes.xyz/ilker/llmgw/internal/version" +) + +// newSelfUpgradeCmd upgrades this machine's llmgw on its own, for the machines +// `llmgw ctl upgrade` does not reach: a mirror host, above all, whose service +// runs unprivileged with the filesystem read-only and so cannot replace its own +// binary the way an agent does. +func newSelfUpgradeCmd() *cobra.Command { + var releases, service string + c := &cobra.Command{ + Use: "upgrade VERSION [--service UNIT]", + Short: "Upgrade this machine's llmgw, and restart the service that runs it", + Long: "For a machine the control plane does not upgrade — the model mirror's host.\n" + + "The control plane and the nodes are upgraded with `llmgw ctl upgrade`.\n\n" + + "Fetches the release for this platform, checks it against the release's\n" + + "checksums, keeps the old binary beside the new one, and restarts --service,\n" + + "waiting for it to be running again. Run it with sudo.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + v := args[0] + out := cmd.OutOrStdout() + if !release.Tag.MatchString(v) { + return fmt.Errorf("%q is not a release version (v1.2.3 or v1.2.3-rc.4)", v) + } + if version.Version == v { + fmt.Fprintf(out, "this machine already runs %s\n", v) + return nil + } + if os.Geteuid() != 0 { + return errors.New("replacing this machine's binary needs root: run it with sudo") + } + self, err := os.Executable() + if err != nil { + return err + } + backup, err := release.Install(cmd.Context(), nil, releases, v, runtime.GOOS, runtime.GOARCH, self) + if err != nil { + return err + } + fmt.Fprintf(out, "%s -> %s: installed, checksum verified; the old binary is %s\n", + version.Version, v, backup) + if service == "" { + return nil + } + //nolint:gosec // the unit name is the operator's own flag + if b, err := exec.CommandContext(cmd.Context(), "systemctl", "restart", service).CombinedOutput(); err != nil { + return fmt.Errorf("restarting %s: %v: %s", service, err, strings.TrimSpace(string(b))) + } + // Running, and still running a few seconds on: a unit that starts + // and dies on a bad flag is "active" for a moment first. + deadline := time.Now().Add(30 * time.Second) + steady := 0 + for time.Now().Before(deadline) { + //nolint:gosec // as above + b, _ := exec.CommandContext(cmd.Context(), "systemctl", "is-active", service).Output() + if strings.TrimSpace(string(b)) == "active" { + if steady++; steady >= 3 { + fmt.Fprintf(out, "%s restarted and running\n", service) + return nil + } + } else { + steady = 0 + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("%s is not running after the upgrade; see journalctl -u %s, and %s holds the previous binary", + service, service, backup) + }, + } + c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from") + c.Flags().StringVar(&service, "service", "", "the systemd unit to restart afterwards, e.g. llmgw-mirror") + return c +}