Upgrade the cluster with one command, over the agents' own streams
Every upgrade so far has been a script of mine copying a binary to each machine over ssh, restarting it and polling until it came back — which is the job, but not something the product could do, and which needed ssh into every node. `sudo llmgw ctl upgrade VERSION` on the control plane does it. This machine first: the release for its platform, checked against the release's checksums, the old binary kept beside it, llmgw-server restarted, and a wait until every node that was ready has reconnected. Then each node in turn: the control plane tells its agent the version over the stream it already holds; the agent fetches and checks the release itself, swaps it in with a rename, answers, and exits with status 75 so systemd starts the new binary. Engines keep running, as for any agent restart, and are adopted again. A node is done when it is back on the new version, ready, with as many ready replicas as before; one that is not within the timeout stops the run before the next node, so a bad release reaches one machine. Nodes already on the version are skipped, so running it again resumes. The fetch-and-verify is one package now, shared with onboarding's idea of what a release version looks like. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
3bf4b7b709
commit
116e75f8cb
19 files changed
+1082
-90
No files matched your search
@@ -8,6 +8,7 @@ import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
@@ -18,6 +19,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -1119,6 +1121,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
|
||||
a.removeWeights(m.RemoveWeights.GetPaths())
|
||||
case *llmgwv1.ControlMessage_Provision:
|
||||
a.setProvision(m.Provision)
|
||||
case *llmgwv1.ControlMessage_Upgrade:
|
||||
go a.upgrade(ctx, m.Upgrade, out)
|
||||
case *llmgwv1.ControlMessage_Logs:
|
||||
reply := a.logsReply(m.Logs)
|
||||
// Off the receive loop: a full send queue must not stop commands being
|
||||
@@ -1874,3 +1878,49 @@ func (a *Agent) sampleNVLink(hb *llmgwv1.Heartbeat) {
|
||||
m := discoverIMEX(ctx)
|
||||
hb.Imex = &llmgwv1.IMEX{State: m.State, Channels: int32(m.Channels)}
|
||||
}
|
||||
|
||||
// upgradeExit is the exit status after an upgrade: non-zero, so systemd's
|
||||
// Restart=on-failure starts the new binary, and distinct, so the journal says
|
||||
// why the agent went.
|
||||
const upgradeExit = 75
|
||||
|
||||
// exitForUpgrade ends the process; a variable so a test can see it called
|
||||
// rather than have its own process end.
|
||||
var exitForUpgrade = func() { os.Exit(upgradeExit) }
|
||||
|
||||
// upgradeTarget is the binary an upgrade replaces; a variable so a test
|
||||
// replaces a file of its own rather than the test binary running it.
|
||||
var upgradeTarget = os.Executable
|
||||
|
||||
// upgrade replaces this agent's binary with a release and restarts into it.
|
||||
//
|
||||
// Engines are not touched: an agent restart leaves them running and adopts
|
||||
// them again, which is what makes upgrading a node something that can be done
|
||||
// while it serves.
|
||||
func (a *Agent) upgrade(ctx context.Context, m *llmgwv1.UpgradeAgent, out chan<- *llmgwv1.AgentMessage) {
|
||||
reply := &llmgwv1.UpgradeReply{RequestId: m.GetRequestId(), Version: m.GetVersion()}
|
||||
send := func() {
|
||||
select {
|
||||
case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: reply}}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}
|
||||
a.log.Info("upgrading", "to", m.GetVersion(), "from", a.cfg.Version)
|
||||
self, err := upgradeTarget()
|
||||
if err == nil {
|
||||
dctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
_, err = release.Install(dctx, nil, m.GetReleases(), m.GetVersion(), runtime.GOOS, runtime.GOARCH, self)
|
||||
cancel()
|
||||
}
|
||||
if err != nil {
|
||||
reply.Error = err.Error()
|
||||
a.log.Error("upgrade failed; still running the old version", "to", m.GetVersion(), "err", err)
|
||||
send()
|
||||
return
|
||||
}
|
||||
send()
|
||||
a.log.Info("upgraded; restarting into the new version", "to", m.GetVersion())
|
||||
// A moment for the reply to leave before the stream goes.
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
exitForUpgrade()
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
)
|
||||
|
||||
func TestAnAgentUpgradesItselfAndSaysSoBeforeRestarting(t *testing.T) {
|
||||
body := []byte("the new agent")
|
||||
sum := sha256.Sum256(body)
|
||||
asset := release.AssetName("v9.9.9", runtime.GOOS, runtime.GOARCH)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/v9.9.9/checksums.txt":
|
||||
_, _ = w.Write([]byte(hex.EncodeToString(sum[:]) + " " + asset + "\n"))
|
||||
case "/v9.9.9/" + asset:
|
||||
_, _ = w.Write(body)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
bin := filepath.Join(t.TempDir(), "llmgw")
|
||||
_ = os.WriteFile(bin, []byte("the old agent"), 0o755)
|
||||
exited := make(chan struct{}, 1)
|
||||
oldTarget, oldExit := upgradeTarget, exitForUpgrade
|
||||
upgradeTarget = func() (string, error) { return bin, nil }
|
||||
exitForUpgrade = func() { exited <- struct{}{} }
|
||||
t.Cleanup(func() { upgradeTarget, exitForUpgrade = oldTarget, oldExit })
|
||||
|
||||
cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(), EnginesDir: t.TempDir()}
|
||||
cfg.Superv.WorkDir = t.TempDir()
|
||||
a, err := New(cfg, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := make(chan *llmgwv1.AgentMessage, 1)
|
||||
go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u1", Version: "v9.9.9", Releases: srv.URL}, out)
|
||||
|
||||
select {
|
||||
case m := <-out:
|
||||
if r := m.GetUpgrade(); r.GetRequestId() != "u1" || r.GetError() != "" {
|
||||
t.Fatalf("reply %+v", r)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("no reply")
|
||||
}
|
||||
select {
|
||||
case <-exited:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the agent did not restart into the new binary")
|
||||
}
|
||||
if b, _ := os.ReadFile(bin); string(b) != "the new agent" {
|
||||
t.Errorf("binary holds %q", b)
|
||||
}
|
||||
|
||||
// A version with no release answers with why, and does not restart.
|
||||
go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u2", Version: "v9.9.8", Releases: srv.URL}, out)
|
||||
if r := (<-out).GetUpgrade(); r.GetError() == "" {
|
||||
t.Error("an upgrade to a release that does not exist reported success")
|
||||
}
|
||||
select {
|
||||
case <-exited:
|
||||
t.Error("a failed upgrade restarted the agent")
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user