Upgrade the cluster with one command, over the agents' own streams

Every upgrade so far has been a script of mine copying a binary to each
machine over ssh, restarting it and polling until it came back — which is
the job, but not something the product could do, and which needed ssh into
every node.

`sudo llmgw ctl upgrade VERSION` on the control plane does it. This machine
first: the release for its platform, checked against the release's
checksums, the old binary kept beside it, llmgw-server restarted, and a wait
until every node that was ready has reconnected. Then each node in turn: the
control plane tells its agent the version over the stream it already holds;
the agent fetches and checks the release itself, swaps it in with a rename,
answers, and exits with status 75 so systemd starts the new binary. Engines
keep running, as for any agent restart, and are adopted again.

A node is done when it is back on the new version, ready, with as many ready
replicas as before; one that is not within the timeout stops the run before
the next node, so a bad release reaches one machine. Nodes already on the
version are skipped, so running it again resumes.

The fetch-and-verify is one package now, shared with onboarding's idea of
what a release version looks like.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 02:54:36 +02:00
1 parent 3bf4b7b709
commit 116e75f8cb
19 files changed
+1082 -90

No files matched your search

+50
View File
@@ -8,6 +8,7 @@ import (
"encoding/hex"
"errors"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"io"
"log/slog"
"maps"
@@ -18,6 +19,7 @@ import (
"os"
"os/exec"
"path/filepath"
"runtime"
"slices"
"strconv"
"strings"
@@ -1119,6 +1121,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
a.removeWeights(m.RemoveWeights.GetPaths())
case *llmgwv1.ControlMessage_Provision:
a.setProvision(m.Provision)
case *llmgwv1.ControlMessage_Upgrade:
go a.upgrade(ctx, m.Upgrade, out)
case *llmgwv1.ControlMessage_Logs:
reply := a.logsReply(m.Logs)
// Off the receive loop: a full send queue must not stop commands being
@@ -1874,3 +1878,49 @@ func (a *Agent) sampleNVLink(hb *llmgwv1.Heartbeat) {
m := discoverIMEX(ctx)
hb.Imex = &llmgwv1.IMEX{State: m.State, Channels: int32(m.Channels)}
}
// upgradeExit is the exit status after an upgrade: non-zero, so systemd's
// Restart=on-failure starts the new binary, and distinct, so the journal says
// why the agent went.
const upgradeExit = 75
// exitForUpgrade ends the process; a variable so a test can see it called
// rather than have its own process end.
var exitForUpgrade = func() { os.Exit(upgradeExit) }
// upgradeTarget is the binary an upgrade replaces; a variable so a test
// replaces a file of its own rather than the test binary running it.
var upgradeTarget = os.Executable
// upgrade replaces this agent's binary with a release and restarts into it.
//
// Engines are not touched: an agent restart leaves them running and adopts
// them again, which is what makes upgrading a node something that can be done
// while it serves.
func (a *Agent) upgrade(ctx context.Context, m *llmgwv1.UpgradeAgent, out chan<- *llmgwv1.AgentMessage) {
reply := &llmgwv1.UpgradeReply{RequestId: m.GetRequestId(), Version: m.GetVersion()}
send := func() {
select {
case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: reply}}:
case <-ctx.Done():
}
}
a.log.Info("upgrading", "to", m.GetVersion(), "from", a.cfg.Version)
self, err := upgradeTarget()
if err == nil {
dctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
_, err = release.Install(dctx, nil, m.GetReleases(), m.GetVersion(), runtime.GOOS, runtime.GOARCH, self)
cancel()
}
if err != nil {
reply.Error = err.Error()
a.log.Error("upgrade failed; still running the old version", "to", m.GetVersion(), "err", err)
send()
return
}
send()
a.log.Info("upgraded; restarting into the new version", "to", m.GetVersion())
// A moment for the reply to leave before the stream goes.
time.Sleep(500 * time.Millisecond)
exitForUpgrade()
}
+78
View File
@@ -0,0 +1,78 @@
package agent
import (
"crypto/sha256"
"encoding/hex"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"testing"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
)
func TestAnAgentUpgradesItselfAndSaysSoBeforeRestarting(t *testing.T) {
body := []byte("the new agent")
sum := sha256.Sum256(body)
asset := release.AssetName("v9.9.9", runtime.GOOS, runtime.GOARCH)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v9.9.9/checksums.txt":
_, _ = w.Write([]byte(hex.EncodeToString(sum[:]) + " " + asset + "\n"))
case "/v9.9.9/" + asset:
_, _ = w.Write(body)
default:
http.NotFound(w, r)
}
}))
defer srv.Close()
bin := filepath.Join(t.TempDir(), "llmgw")
_ = os.WriteFile(bin, []byte("the old agent"), 0o755)
exited := make(chan struct{}, 1)
oldTarget, oldExit := upgradeTarget, exitForUpgrade
upgradeTarget = func() (string, error) { return bin, nil }
exitForUpgrade = func() { exited <- struct{}{} }
t.Cleanup(func() { upgradeTarget, exitForUpgrade = oldTarget, oldExit })
cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(), EnginesDir: t.TempDir()}
cfg.Superv.WorkDir = t.TempDir()
a, err := New(cfg, nil, nil)
if err != nil {
t.Fatal(err)
}
out := make(chan *llmgwv1.AgentMessage, 1)
go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u1", Version: "v9.9.9", Releases: srv.URL}, out)
select {
case m := <-out:
if r := m.GetUpgrade(); r.GetRequestId() != "u1" || r.GetError() != "" {
t.Fatalf("reply %+v", r)
}
case <-time.After(10 * time.Second):
t.Fatal("no reply")
}
select {
case <-exited:
case <-time.After(5 * time.Second):
t.Fatal("the agent did not restart into the new binary")
}
if b, _ := os.ReadFile(bin); string(b) != "the new agent" {
t.Errorf("binary holds %q", b)
}
// A version with no release answers with why, and does not restart.
go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u2", Version: "v9.9.8", Releases: srv.URL}, out)
if r := (<-out).GetUpgrade(); r.GetError() == "" {
t.Error("an upgrade to a release that does not exist reported success")
}
select {
case <-exited:
t.Error("a failed upgrade restarted the agent")
case <-time.After(time.Second):
}
}
+1
View File
@@ -141,6 +141,7 @@ func newCtlCmd() *cobra.Command {
newScaleCmd(client),
newRestartCmd(client),
newRetryCmd(client),
newUpgradeCmd(client),
newEngineCheckCmd(),
newToolCheckCmd(),
newWeightsCmd(client, &asJSON),
+197
View File
@@ -0,0 +1,197 @@
package cli
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"runtime"
"slices"
"strings"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
func newUpgradeCmd(client func() *ctlClient) *cobra.Command {
var (
nodes []string
skipCP bool
releases string
perNode time.Duration
serverSvc string
)
c := &cobra.Command{
Use: "upgrade VERSION",
Short: "Upgrade this control plane, then every node, one at a time",
Long: "Moves the cluster to a release: v0.10.4, or a candidate like v0.10.4-rc.2.\n\n" +
"This machine first — run it with sudo on the control plane, since it\n" +
"replaces /usr/local/bin/llmgw and restarts llmgw-server. Then each node, in\n" +
"turn: its agent fetches the release for its own platform, checks it\n" +
"against the release's checksums, swaps it in and restarts. Engines keep\n" +
"serving through all of it and are adopted again; no ssh into the nodes is\n" +
"needed.\n\n" +
"A node counts as done when it is back on the new version, ready, with as\n" +
"many ready replicas as before. One that is not within --timeout stops the\n" +
"upgrade there, so a bad release reaches one node rather than all of them.\n" +
"Nodes already on the version are skipped, so running it again resumes.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
v := args[0]
if !release.Tag.MatchString(v) {
return fmt.Errorf("%q is not a release version (v1.2.3 or v1.2.3-rc.4)", v)
}
out := cmd.OutOrStdout()
ctx := cmd.Context()
if !skipCP {
if err := upgradeControlPlane(ctx, out, client, v, releases, serverSvc); err != nil {
return err
}
}
return upgradeNodes(ctx, out, client, v, nodes, perNode)
},
}
c.Flags().StringSliceVar(&nodes, "nodes", nil, "only these nodes (default: all)")
c.Flags().BoolVar(&skipCP, "skip-control-plane", false, "leave this machine as it is and upgrade only nodes")
c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from, for this machine")
c.Flags().DurationVar(&perNode, "timeout", 3*time.Minute, "how long a node may take to come back")
c.Flags().StringVar(&serverSvc, "service", "llmgw-server", "this machine's systemd unit")
return c
}
func upgradeControlPlane(ctx context.Context, out io.Writer, client func() *ctlClient, v, releases, svc string) error {
if version.Version == v {
fmt.Fprintf(out, "this machine already runs %s\n", v)
return nil
}
if os.Geteuid() != 0 {
return errors.New("replacing this machine's binary needs root: run it with sudo, " +
"or pass --skip-control-plane to upgrade only the nodes")
}
self, err := os.Executable()
if err != nil {
return err
}
fmt.Fprintf(out, "== this machine: %s -> %s\n", version.Version, v)
backup, err := release.Install(ctx, nil, releases, v, runtime.GOOS, runtime.GOARCH, self)
if err != nil {
return err
}
fmt.Fprintf(out, " installed, checksum verified; the old binary is %s\n", backup)
var before []adminapi.NodeView
_ = client().get(ctx, "/api/v1/nodes", &before)
if b, err := exec.CommandContext(ctx, "systemctl", "restart", svc).CombinedOutput(); err != nil { //nolint:gosec // fixed command
return fmt.Errorf("restarting %s: %v: %s", svc, err, strings.TrimSpace(string(b)))
}
fmt.Fprintf(out, " %s restarted; waiting for its nodes to reconnect\n", svc)
// Back when the admin API answers and every node that was ready is ready
// again: agents reconnect within seconds of the control plane returning.
want := 0
for _, n := range before {
if n.State == "ready" {
want++
}
}
deadline := time.Now().Add(2 * time.Minute)
for {
var now []adminapi.NodeView
if err := client().get(ctx, "/api/v1/nodes", &now); err == nil {
ready := 0
for _, n := range now {
if n.State == "ready" {
ready++
}
}
if ready >= want {
fmt.Fprintf(out, " back: %d of %d nodes ready\n", ready, len(now))
return nil
}
}
if time.Now().After(deadline) {
return fmt.Errorf("%s did not come back with its %d nodes within two minutes; "+
"see journalctl -u %s, and %s holds the previous binary", svc, want, svc, backup)
}
time.Sleep(2 * time.Second)
}
}
func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient, v string, only []string, perNode time.Duration) error {
var nodes []adminapi.NodeView
if err := client().get(ctx, "/api/v1/nodes", &nodes); err != nil {
return err
}
slices.SortFunc(nodes, func(a, b adminapi.NodeView) int { return strings.Compare(a.ID, b.ID) })
done, skipped := 0, 0
for _, n := range nodes {
if len(only) > 0 && !slices.Contains(only, n.ID) {
continue
}
if n.AgentVersion == v {
skipped++
continue
}
start := time.Now()
before := readyReplicasOn(ctx, client, n.ID)
fmt.Fprintf(out, "== %s: %s -> %s\n", n.ID, n.AgentVersion, v)
body, _ := json.Marshal(adminapi.UpgradeRequest{Version: v})
if err := client().post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil {
return fmt.Errorf("%s: %w\nstopped here; `llmgw ctl logs %s` says more", n.ID, err, n.ID)
}
fmt.Fprintf(out, " new binary in place; waiting for it to come back with its %d replica(s)\n", before)
deadline := time.Now().Add(perNode)
for {
var now []adminapi.NodeView
_ = client().get(ctx, "/api/v1/nodes", &now)
var cur adminapi.NodeView
for _, x := range now {
if x.ID == n.ID {
cur = x
}
}
after := readyReplicasOn(ctx, client, n.ID)
if cur.AgentVersion == v && cur.State == "ready" && after >= before {
fmt.Fprintf(out, " done in %s: ready, replicas %d/%d\n",
time.Since(start).Round(time.Second), after, before)
done++
break
}
if time.Now().After(deadline) {
return fmt.Errorf("%s did not come back within %s (version %q, state %q, replicas %d/%d)\n"+
"stopped here, before the next node; `llmgw ctl logs %s` says why, and the node "+
"keeps its previous binary beside the new one", n.ID, perNode, cur.AgentVersion,
cur.State, after, before, n.ID)
}
time.Sleep(2 * time.Second)
}
}
fmt.Fprintf(out, "%d node(s) upgraded to %s, %d already on it\n", done, v, skipped)
return nil
}
// readyReplicasOn counts the ready replicas with a shard on a node.
func readyReplicasOn(ctx context.Context, client func() *ctlClient, node string) int {
var reps []adminapi.ReplicaView
if err := client().get(ctx, "/api/v1/replicas", &reps); err != nil {
return 0
}
n := 0
for _, r := range reps {
if r.State != "ready" {
continue
}
for _, s := range r.Shards {
if s.Node == node {
n++
break
}
}
}
return n
}
+88
View File
@@ -0,0 +1,88 @@
package cli
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
// fakeFleet is an admin API whose nodes come back on the version they are told
// to upgrade to — except those listed in stuck.
type fakeFleet struct {
mu sync.Mutex
versions map[string]string
stuck map[string]bool
told []string
}
func (f *fakeFleet) server(t *testing.T) *httptest.Server {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
f.mu.Lock()
defer f.mu.Unlock()
switch {
case r.URL.Path == "/api/v1/nodes":
var out []adminapi.NodeView
for id, v := range f.versions {
out = append(out, adminapi.NodeView{ID: id, State: "ready", AgentVersion: v})
}
_ = json.NewEncoder(w).Encode(out)
case r.URL.Path == "/api/v1/replicas":
var out []adminapi.ReplicaView
for id := range f.versions {
out = append(out, adminapi.ReplicaView{ID: "r-" + id, State: "ready",
Shards: []adminapi.ShardView{{Node: id}}})
}
_ = json.NewEncoder(w).Encode(out)
case strings.HasSuffix(r.URL.Path, "/upgrade"):
id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/nodes/"), "/upgrade")
var req adminapi.UpgradeRequest
_ = json.NewDecoder(r.Body).Decode(&req)
f.told = append(f.told, id)
if !f.stuck[id] {
f.versions[id] = req.Version
}
_, _ = w.Write([]byte(`{}`))
default:
http.NotFound(w, r)
}
}))
t.Cleanup(srv.Close)
return srv
}
func TestAnUpgradeGoesNodeByNodeAndSkipsTheDone(t *testing.T) {
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.1", "c": "v1.0.0"}}
srv := f.server(t)
var out bytes.Buffer
err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 5*time.Second)
if err != nil {
t.Fatal(err)
}
if strings.Join(f.told, ",") != "a,c" {
t.Errorf("told %v; b was already on the version", f.told)
}
if !strings.Contains(out.String(), "2 node(s) upgraded to v1.0.1, 1 already on it") {
t.Errorf("output:\n%s", out.String())
}
}
func TestAnUpgradeStopsAtTheFirstNodeThatDoesNotComeBack(t *testing.T) {
// A bad release reaches one node, not all of them.
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.0"}, stuck: map[string]bool{"a": true}}
srv := f.server(t)
var out bytes.Buffer
err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 3*time.Second)
if err == nil || !strings.Contains(err.Error(), "a did not come back") {
t.Fatalf("err %v", err)
}
if len(f.told) != 1 {
t.Errorf("went on to %v after a node failed to come back", f.told)
}
}
+14
View File
@@ -173,6 +173,7 @@ func newServerCmd() *cobra.Command {
}
if admin != nil {
api := adminapi.New(admin)
api.SetReleases(cfg.Releases)
// Catalog changes need somewhere to persist, so this is wired
// only when there is a store. Without one, the endpoints say so
// rather than accepting a model that vanishes on restart.
@@ -546,6 +547,19 @@ func (r *dispatcherRef) FabricTest(ctx context.Context, nodeID string, req *llmg
return ft.FabricTest(ctx, nodeID, req)
}
// Upgrade forwards a node upgrade, as Logs does.
func (r *dispatcherRef) Upgrade(ctx context.Context, nodeID, version, releases string) error {
d, err := r.get()
if err != nil {
return err
}
u, ok := d.(control.Upgrader)
if !ok {
return errors.New("this dispatcher cannot upgrade nodes")
}
return u.Upgrade(ctx, nodeID, version, releases)
}
func (r *dispatcherRef) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
d, err := r.get()
if err != nil {
+5
View File
@@ -86,6 +86,11 @@ type Server struct {
// Empty means /var/backups/llmgw-server.
BackupDir string `koanf:"backup_dir"`
// Releases is where `llmgw ctl upgrade` fetches release binaries from,
// for this machine and — through their agents — every node. Empty means
// the project's own releases.
Releases string `koanf:"releases"`
Router Router `koanf:"router"`
Control Control `koanf:"control"`
Models []catalog.Model `koanf:"models"`
+39
View File
@@ -10,6 +10,7 @@ import (
"context"
"encoding/json"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"io"
"net/http"
"slices"
@@ -30,6 +31,9 @@ import (
// API serves the management endpoints.
type API struct {
// releases is where release binaries are fetched from, for upgrades.
releases string
models Models
capacity Capacity
usage Usage
@@ -74,6 +78,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /api/v1/weights/collect", a.collectWeights)
mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement)
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
mux.HandleFunc("POST /api/v1/nodes/{id}/upgrade", a.upgradeNode)
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
@@ -1165,3 +1170,37 @@ func (a *API) request(w http.ResponseWriter, r *http.Request) {
PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros,
})
}
// SetReleases sets where upgrades fetch release binaries from.
func (a *API) SetReleases(base string) { a.releases = base }
// UpgradeRequest names the release a node should run.
type UpgradeRequest struct {
Version string `json:"version"`
}
func (a *API) upgradeNode(w http.ResponseWriter, r *http.Request) {
var req UpgradeRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the upgrade request is not valid JSON: "+err.Error())
return
}
if !release.Tag.MatchString(req.Version) {
writeError(w, http.StatusBadRequest, req.Version+" is not a release version")
return
}
u := a.ctl.Upgrader()
if u == nil {
writeError(w, http.StatusNotImplemented, "this control plane cannot upgrade nodes")
return
}
base := a.releases
if base == "" {
base = release.DefaultBase
}
if err := u.Upgrade(r.Context(), r.PathValue("id"), req.Version, base); err != nil {
writeError(w, http.StatusBadGateway, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"node": r.PathValue("id"), "version": req.Version})
}
@@ -264,3 +264,22 @@ func TestAGPUFallingOutOfItsDomainTakesTheNodeOut(t *testing.T) {
t.Errorf("imex = %+v", n.IMEX)
}
}
func TestAnUpgradeWaitsForTheAgentsAnswer(t *testing.T) {
s := newTestService()
c := &conn{nodeID: "n1", send: make(chan *llmgwv1.ControlMessage, 4), closed: make(chan struct{})}
s.conns["n1"] = c
done := make(chan error, 1)
go func() { done <- s.Upgrade(t.Context(), "n1", "v1.2.3", "https://r") }()
req := (<-c.send).GetUpgrade()
if req.GetVersion() != "v1.2.3" || req.GetReleases() != "https://r" {
t.Fatalf("sent %v", req)
}
s.handle("n1", &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: &llmgwv1.UpgradeReply{
RequestId: req.GetRequestId(), Version: "v1.2.3", Error: "checksum mismatch",
}}})
if err := <-done; err == nil || err.Error() != "checksum mismatch" {
t.Errorf("the agent's refusal was not passed on: %v", err)
}
}
+52
View File
@@ -60,6 +60,8 @@ type Service struct {
logMu sync.Mutex
logWaits map[string]chan *llmgwv1.LogsReply
logSeq uint64
// upgradeWaits are the upgrade requests awaiting a reply; guarded by logMu.
upgradeWaits map[string]chan *llmgwv1.UpgradeReply
// scanWaits are the fabric scans awaiting a reply, by request id.
scanWaits map[string]chan *llmgwv1.FabricScanReply
testWaits map[string]chan *llmgwv1.FabricTestReply
@@ -309,6 +311,15 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
ch <- m.FabricScan
}
case *llmgwv1.AgentMessage_Upgrade:
s.logMu.Lock()
ch, ok := s.upgradeWaits[m.Upgrade.GetRequestId()]
delete(s.upgradeWaits, m.Upgrade.GetRequestId())
s.logMu.Unlock()
if ok {
ch <- m.Upgrade
}
case *llmgwv1.AgentMessage_Logs:
s.logMu.Lock()
ch, ok := s.logWaits[m.Logs.GetRequestId()]
@@ -842,3 +853,44 @@ func installable(reg *llmgwv1.Register) []string {
// The admin API finds the bandwidth test by asserting this; a signature that
// drifted would make it quietly nil.
var _ fabrictest.Asker = (*Service)(nil)
// Upgrade tells a node's agent to replace its binary with a release and
// restart, and returns once the agent says the new binary is in place — or
// why it is not. The agent then exits and comes back on the new version; the
// caller watches for that in the inventory.
func (s *Service) Upgrade(ctx context.Context, nodeID, version, releases string) error {
s.logMu.Lock()
if s.upgradeWaits == nil {
s.upgradeWaits = map[string]chan *llmgwv1.UpgradeReply{}
}
s.logSeq++
id := fmt.Sprintf("upgrade-%d", s.logSeq)
ch := make(chan *llmgwv1.UpgradeReply, 1)
s.upgradeWaits[id] = ch
s.logMu.Unlock()
defer func() {
s.logMu.Lock()
delete(s.upgradeWaits, id)
s.logMu.Unlock()
}()
if err := s.send(nodeID, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_Upgrade{
Upgrade: &llmgwv1.UpgradeAgent{RequestId: id, Version: version, Releases: releases},
}}); err != nil {
return err
}
s.log.Info("told a node to upgrade", "node", nodeID, "version", version)
// A download of some tens of megabytes on a slow link, and no longer: an
// agent too old to know this message never answers.
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
select {
case r := <-ch:
if r.GetError() != "" {
return errors.New(r.GetError())
}
return nil
case <-ctx.Done():
return fmt.Errorf("node %s did not answer the upgrade; an agent older than v0.10.4 cannot upgrade itself", nodeID)
}
}
+11
View File
@@ -211,6 +211,17 @@ func (c *Controller) WeightRemover() WeightRemover {
return r
}
// Upgrader can tell a node's agent to upgrade itself.
type Upgrader interface {
Upgrade(ctx context.Context, nodeID, version, releases string) error
}
// Upgrader returns the dispatcher's upgrade capability, or nil.
func (c *Controller) Upgrader() Upgrader {
u, _ := c.disp.(Upgrader)
return u
}
// LogReader can ask a node for the recent lines of one of its logs.
type LogReader interface {
Logs(ctx context.Context, nodeID, source string, lines int) (out, sources []string, err error)
+2 -1
View File
@@ -18,6 +18,7 @@ import (
"encoding/pem"
"errors"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"net"
"os"
"path/filepath"
@@ -335,7 +336,7 @@ func certNames(path string) ([]string, error) {
// releaseTag is a version CI published: v0.10.4, or a release candidate of
// it, v0.10.4-rc.2. Anything else — a build with commits past its tag — has no
// release to download.
var releaseTag = regexp.MustCompile(`^v\d+\.\d+\.\d+(-rc\.\d+)?$`)
var releaseTag = release.Tag
func (r *Runner) binary(ctx context.Context, res *Result) error {
v := r.Opts.Version
+148
View File
@@ -0,0 +1,148 @@
// Package release fetches and installs llmgw's own release binaries.
//
// One implementation for everywhere a binary is replaced — the control plane
// upgrading itself, and an agent told to upgrade by it — so the checks are
// the same on both: the release's own checksum is verified before anything is
// installed, the old binary is kept beside the new one, and the swap is a
// rename, so a machine is never left without a binary that runs.
package release
import (
"bufio"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"time"
)
// DefaultBase is where CI publishes releases.
const DefaultBase = "https://gitea.mynodes.xyz/ilker/llmgw/releases/download"
// Tag is a version CI published: v0.10.4, or a release candidate v0.10.4-rc.2.
var Tag = regexp.MustCompile(`^v\d+\.\d+\.\d+(-rc\.\d+)?$`)
// AssetName is the release file for a platform.
func AssetName(version, goos, goarch string) string {
return "llmgw-" + version + "-" + goos + "-" + goarch
}
// Install fetches version for this platform from base, checks it against the
// release's checksums, and puts it at target, keeping the old one as
// target.bak-pre-<version>. It returns the backup's path.
func Install(ctx context.Context, client *http.Client, base, version, goos, goarch, target string) (string, error) {
if !Tag.MatchString(version) {
return "", fmt.Errorf("%q is not a release version", version)
}
if client == nil {
client = &http.Client{Timeout: 10 * time.Minute}
}
target, err := filepath.EvalSymlinks(target)
if err != nil {
return "", err
}
dir := filepath.Dir(target)
asset := AssetName(version, goos, goarch)
root := strings.TrimSuffix(base, "/") + "/" + version + "/"
want, err := checksum(ctx, client, root+"checksums.txt", asset)
if err != nil {
return "", err
}
// Beside the target, so the final rename is on one filesystem and atomic.
tmp, err := os.CreateTemp(dir, ".llmgw-"+version+"-*")
if err != nil {
return "", err
}
defer os.Remove(tmp.Name())
got, err := download(ctx, client, root+asset, tmp)
if cerr := tmp.Close(); err == nil {
err = cerr
}
if err != nil {
return "", err
}
if got != want {
return "", fmt.Errorf("%s does not match the release's checksum (got %s, expected %s)",
asset, got[:12], want[:12])
}
if err := os.Chmod(tmp.Name(), 0o755); err != nil {
return "", err
}
backup := target + ".bak-pre-" + version
if err := copyFile(target, backup); err != nil {
return "", fmt.Errorf("keeping the old binary: %w", err)
}
if err := os.Rename(tmp.Name(), target); err != nil {
return "", err
}
return backup, nil
}
func checksum(ctx context.Context, client *http.Client, url, asset string) (string, error) {
body, err := get(ctx, client, url)
if err != nil {
return "", err
}
defer body.Close()
sc := bufio.NewScanner(io.LimitReader(body, 1<<20))
for sc.Scan() {
f := strings.Fields(sc.Text())
if len(f) == 2 && f[1] == asset && len(f[0]) == 64 {
return f[0], nil
}
}
return "", fmt.Errorf("the release's checksums list no %s", asset)
}
func download(ctx context.Context, client *http.Client, url string, dst io.Writer) (string, error) {
body, err := get(ctx, client, url)
if err != nil {
return "", err
}
defer body.Close()
h := sha256.New()
if _, err := io.Copy(io.MultiWriter(dst, h), body); err != nil {
return "", fmt.Errorf("downloading %s: %w", url, err)
}
return hex.EncodeToString(h.Sum(nil)), nil
}
func get(ctx context.Context, client *http.Client, url string) (io.ReadCloser, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("fetching %s: %w", url, err)
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("fetching %s: %s", url, resp.Status)
}
return resp.Body, nil
}
func copyFile(src, dst string) error {
in, err := os.Open(src) //nolint:gosec // our own binary
if err != nil {
return err
}
defer in.Close()
out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755) //nolint:gosec // beside it
if err != nil {
return err
}
if _, err := io.Copy(out, in); err != nil {
out.Close()
return err
}
return out.Close()
}
+61
View File
@@ -0,0 +1,61 @@
package release
import (
"crypto/sha256"
"encoding/hex"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
)
func releaseServer(t *testing.T, version, asset string, body []byte, sum string) *httptest.Server {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/" + version + "/checksums.txt":
_, _ = w.Write([]byte(sum + " " + asset + "\n"))
case "/" + version + "/" + asset:
_, _ = w.Write(body)
default:
http.NotFound(w, r)
}
}))
t.Cleanup(srv.Close)
return srv
}
func TestABinaryIsReplacedOnlyWithWhatTheReleasePublished(t *testing.T) {
dir := t.TempDir()
target := filepath.Join(dir, "llmgw")
_ = os.WriteFile(target, []byte("old"), 0o755)
body := []byte("new binary")
sum := sha256.Sum256(body)
asset := AssetName("v1.2.3", "linux", "amd64")
srv := releaseServer(t, "v1.2.3", asset, body, hex.EncodeToString(sum[:]))
backup, err := Install(t.Context(), srv.Client(), srv.URL, "v1.2.3", "linux", "amd64", target)
if err != nil {
t.Fatal(err)
}
if b, _ := os.ReadFile(target); string(b) != "new binary" {
t.Errorf("target holds %q", b)
}
if b, _ := os.ReadFile(backup); string(b) != "old" {
t.Errorf("the old binary was not kept: %q", b)
}
// A download that does not match its checksum changes nothing.
bad := releaseServer(t, "v1.2.4", AssetName("v1.2.4", "linux", "amd64"), []byte("tampered"),
hex.EncodeToString(sum[:]))
if _, err := Install(t.Context(), bad.Client(), bad.URL, "v1.2.4", "linux", "amd64", target); err == nil {
t.Fatal("a binary not matching its checksum was installed")
}
if b, _ := os.ReadFile(target); string(b) != "new binary" {
t.Errorf("a failed install touched the binary: %q", b)
}
if _, err := Install(t.Context(), nil, srv.URL, "../../x", "linux", "amd64", target); err == nil {
t.Error("a version that is not one was fetched")
}
}