Every upgrade so far has been a script of mine copying a binary to each machine over ssh, restarting it and polling until it came back — which is the job, but not something the product could do, and which needed ssh into every node. `sudo llmgw ctl upgrade VERSION` on the control plane does it. This machine first: the release for its platform, checked against the release's checksums, the old binary kept beside it, llmgw-server restarted, and a wait until every node that was ready has reconnected. Then each node in turn: the control plane tells its agent the version over the stream it already holds; the agent fetches and checks the release itself, swaps it in with a rename, answers, and exits with status 75 so systemd starts the new binary. Engines keep running, as for any agent restart, and are adopted again. A node is done when it is back on the new version, ready, with as many ready replicas as before; one that is not within the timeout stops the run before the next node, so a bad release reaches one machine. Nodes already on the version are skipped, so running it again resumes. The fetch-and-verify is one package now, shared with onboarding's idea of what a release version looks like. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
149 lines
4.2 KiB
Go
149 lines
4.2 KiB
Go
// Package release fetches and installs llmgw's own release binaries.
|
|
//
|
|
// One implementation for everywhere a binary is replaced — the control plane
|
|
// upgrading itself, and an agent told to upgrade by it — so the checks are
|
|
// the same on both: the release's own checksum is verified before anything is
|
|
// installed, the old binary is kept beside the new one, and the swap is a
|
|
// rename, so a machine is never left without a binary that runs.
|
|
package release
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// DefaultBase is where CI publishes releases.
|
|
const DefaultBase = "https://gitea.mynodes.xyz/ilker/llmgw/releases/download"
|
|
|
|
// Tag is a version CI published: v0.10.4, or a release candidate v0.10.4-rc.2.
|
|
var Tag = regexp.MustCompile(`^v\d+\.\d+\.\d+(-rc\.\d+)?$`)
|
|
|
|
// AssetName is the release file for a platform.
|
|
func AssetName(version, goos, goarch string) string {
|
|
return "llmgw-" + version + "-" + goos + "-" + goarch
|
|
}
|
|
|
|
// Install fetches version for this platform from base, checks it against the
|
|
// release's checksums, and puts it at target, keeping the old one as
|
|
// target.bak-pre-<version>. It returns the backup's path.
|
|
func Install(ctx context.Context, client *http.Client, base, version, goos, goarch, target string) (string, error) {
|
|
if !Tag.MatchString(version) {
|
|
return "", fmt.Errorf("%q is not a release version", version)
|
|
}
|
|
if client == nil {
|
|
client = &http.Client{Timeout: 10 * time.Minute}
|
|
}
|
|
target, err := filepath.EvalSymlinks(target)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
dir := filepath.Dir(target)
|
|
asset := AssetName(version, goos, goarch)
|
|
root := strings.TrimSuffix(base, "/") + "/" + version + "/"
|
|
|
|
want, err := checksum(ctx, client, root+"checksums.txt", asset)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
// Beside the target, so the final rename is on one filesystem and atomic.
|
|
tmp, err := os.CreateTemp(dir, ".llmgw-"+version+"-*")
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer os.Remove(tmp.Name())
|
|
got, err := download(ctx, client, root+asset, tmp)
|
|
if cerr := tmp.Close(); err == nil {
|
|
err = cerr
|
|
}
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if got != want {
|
|
return "", fmt.Errorf("%s does not match the release's checksum (got %s, expected %s)",
|
|
asset, got[:12], want[:12])
|
|
}
|
|
if err := os.Chmod(tmp.Name(), 0o755); err != nil {
|
|
return "", err
|
|
}
|
|
backup := target + ".bak-pre-" + version
|
|
if err := copyFile(target, backup); err != nil {
|
|
return "", fmt.Errorf("keeping the old binary: %w", err)
|
|
}
|
|
if err := os.Rename(tmp.Name(), target); err != nil {
|
|
return "", err
|
|
}
|
|
return backup, nil
|
|
}
|
|
|
|
func checksum(ctx context.Context, client *http.Client, url, asset string) (string, error) {
|
|
body, err := get(ctx, client, url)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer body.Close()
|
|
sc := bufio.NewScanner(io.LimitReader(body, 1<<20))
|
|
for sc.Scan() {
|
|
f := strings.Fields(sc.Text())
|
|
if len(f) == 2 && f[1] == asset && len(f[0]) == 64 {
|
|
return f[0], nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("the release's checksums list no %s", asset)
|
|
}
|
|
|
|
func download(ctx context.Context, client *http.Client, url string, dst io.Writer) (string, error) {
|
|
body, err := get(ctx, client, url)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer body.Close()
|
|
h := sha256.New()
|
|
if _, err := io.Copy(io.MultiWriter(dst, h), body); err != nil {
|
|
return "", fmt.Errorf("downloading %s: %w", url, err)
|
|
}
|
|
return hex.EncodeToString(h.Sum(nil)), nil
|
|
}
|
|
|
|
func get(ctx context.Context, client *http.Client, url string) (io.ReadCloser, error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("fetching %s: %w", url, err)
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
resp.Body.Close()
|
|
return nil, fmt.Errorf("fetching %s: %s", url, resp.Status)
|
|
}
|
|
return resp.Body, nil
|
|
}
|
|
|
|
func copyFile(src, dst string) error {
|
|
in, err := os.Open(src) //nolint:gosec // our own binary
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer in.Close()
|
|
out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755) //nolint:gosec // beside it
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := io.Copy(out, in); err != nil {
|
|
out.Close()
|
|
return err
|
|
}
|
|
return out.Close()
|
|
}
|