Files
llmgw/internal/release/release.go
T
Ilker ManapandClaude Opus 5.5 116e75f8cb Upgrade the cluster with one command, over the agents' own streams
Every upgrade so far has been a script of mine copying a binary to each
machine over ssh, restarting it and polling until it came back — which is
the job, but not something the product could do, and which needed ssh into
every node.

`sudo llmgw ctl upgrade VERSION` on the control plane does it. This machine
first: the release for its platform, checked against the release's
checksums, the old binary kept beside it, llmgw-server restarted, and a wait
until every node that was ready has reconnected. Then each node in turn: the
control plane tells its agent the version over the stream it already holds;
the agent fetches and checks the release itself, swaps it in with a rename,
answers, and exits with status 75 so systemd starts the new binary. Engines
keep running, as for any agent restart, and are adopted again.

A node is done when it is back on the new version, ready, with as many ready
replicas as before; one that is not within the timeout stops the run before
the next node, so a bad release reaches one machine. Nodes already on the
version are skipped, so running it again resumes.

The fetch-and-verify is one package now, shared with onboarding's idea of
what a release version looks like.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 02:54:36 +02:00

149 lines
4.2 KiB
Go

// Package release fetches and installs llmgw's own release binaries.
//
// One implementation for everywhere a binary is replaced — the control plane
// upgrading itself, and an agent told to upgrade by it — so the checks are
// the same on both: the release's own checksum is verified before anything is
// installed, the old binary is kept beside the new one, and the swap is a
// rename, so a machine is never left without a binary that runs.
package release
import (
"bufio"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"time"
)
// DefaultBase is where CI publishes releases.
const DefaultBase = "https://gitea.mynodes.xyz/ilker/llmgw/releases/download"
// Tag is a version CI published: v0.10.4, or a release candidate v0.10.4-rc.2.
var Tag = regexp.MustCompile(`^v\d+\.\d+\.\d+(-rc\.\d+)?$`)
// AssetName is the release file for a platform.
func AssetName(version, goos, goarch string) string {
return "llmgw-" + version + "-" + goos + "-" + goarch
}
// Install fetches version for this platform from base, checks it against the
// release's checksums, and puts it at target, keeping the old one as
// target.bak-pre-<version>. It returns the backup's path.
func Install(ctx context.Context, client *http.Client, base, version, goos, goarch, target string) (string, error) {
if !Tag.MatchString(version) {
return "", fmt.Errorf("%q is not a release version", version)
}
if client == nil {
client = &http.Client{Timeout: 10 * time.Minute}
}
target, err := filepath.EvalSymlinks(target)
if err != nil {
return "", err
}
dir := filepath.Dir(target)
asset := AssetName(version, goos, goarch)
root := strings.TrimSuffix(base, "/") + "/" + version + "/"
want, err := checksum(ctx, client, root+"checksums.txt", asset)
if err != nil {
return "", err
}
// Beside the target, so the final rename is on one filesystem and atomic.
tmp, err := os.CreateTemp(dir, ".llmgw-"+version+"-*")
if err != nil {
return "", err
}
defer os.Remove(tmp.Name())
got, err := download(ctx, client, root+asset, tmp)
if cerr := tmp.Close(); err == nil {
err = cerr
}
if err != nil {
return "", err
}
if got != want {
return "", fmt.Errorf("%s does not match the release's checksum (got %s, expected %s)",
asset, got[:12], want[:12])
}
if err := os.Chmod(tmp.Name(), 0o755); err != nil {
return "", err
}
backup := target + ".bak-pre-" + version
if err := copyFile(target, backup); err != nil {
return "", fmt.Errorf("keeping the old binary: %w", err)
}
if err := os.Rename(tmp.Name(), target); err != nil {
return "", err
}
return backup, nil
}
func checksum(ctx context.Context, client *http.Client, url, asset string) (string, error) {
body, err := get(ctx, client, url)
if err != nil {
return "", err
}
defer body.Close()
sc := bufio.NewScanner(io.LimitReader(body, 1<<20))
for sc.Scan() {
f := strings.Fields(sc.Text())
if len(f) == 2 && f[1] == asset && len(f[0]) == 64 {
return f[0], nil
}
}
return "", fmt.Errorf("the release's checksums list no %s", asset)
}
func download(ctx context.Context, client *http.Client, url string, dst io.Writer) (string, error) {
body, err := get(ctx, client, url)
if err != nil {
return "", err
}
defer body.Close()
h := sha256.New()
if _, err := io.Copy(io.MultiWriter(dst, h), body); err != nil {
return "", fmt.Errorf("downloading %s: %w", url, err)
}
return hex.EncodeToString(h.Sum(nil)), nil
}
func get(ctx context.Context, client *http.Client, url string) (io.ReadCloser, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("fetching %s: %w", url, err)
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("fetching %s: %s", url, resp.Status)
}
return resp.Body, nil
}
func copyFile(src, dst string) error {
in, err := os.Open(src) //nolint:gosec // our own binary
if err != nil {
return err
}
defer in.Close()
out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755) //nolint:gosec // beside it
if err != nil {
return err
}
if _, err := io.Copy(out, in); err != nil {
out.Close()
return err
}
return out.Close()
}