Files
llmgw/internal/agent/upgrade_test.go
T
Ilker ManapandClaude Opus 5.5 116e75f8cb
Release / build (push) Successful in 1m5s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m42s
Upgrade the cluster with one command, over the agents' own streams
Every upgrade so far has been a script of mine copying a binary to each
machine over ssh, restarting it and polling until it came back — which is
the job, but not something the product could do, and which needed ssh into
every node.

`sudo llmgw ctl upgrade VERSION` on the control plane does it. This machine
first: the release for its platform, checked against the release's
checksums, the old binary kept beside it, llmgw-server restarted, and a wait
until every node that was ready has reconnected. Then each node in turn: the
control plane tells its agent the version over the stream it already holds;
the agent fetches and checks the release itself, swaps it in with a rename,
answers, and exits with status 75 so systemd starts the new binary. Engines
keep running, as for any agent restart, and are adopted again.

A node is done when it is back on the new version, ready, with as many ready
replicas as before; one that is not within the timeout stops the run before
the next node, so a bad release reaches one machine. Nodes already on the
version are skipped, so running it again resumes.

The fetch-and-verify is one package now, shared with onboarding's idea of
what a release version looks like.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 02:54:36 +02:00

79 lines
2.3 KiB
Go

package agent
import (
"crypto/sha256"
"encoding/hex"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"testing"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
)
func TestAnAgentUpgradesItselfAndSaysSoBeforeRestarting(t *testing.T) {
body := []byte("the new agent")
sum := sha256.Sum256(body)
asset := release.AssetName("v9.9.9", runtime.GOOS, runtime.GOARCH)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v9.9.9/checksums.txt":
_, _ = w.Write([]byte(hex.EncodeToString(sum[:]) + " " + asset + "\n"))
case "/v9.9.9/" + asset:
_, _ = w.Write(body)
default:
http.NotFound(w, r)
}
}))
defer srv.Close()
bin := filepath.Join(t.TempDir(), "llmgw")
_ = os.WriteFile(bin, []byte("the old agent"), 0o755)
exited := make(chan struct{}, 1)
oldTarget, oldExit := upgradeTarget, exitForUpgrade
upgradeTarget = func() (string, error) { return bin, nil }
exitForUpgrade = func() { exited <- struct{}{} }
t.Cleanup(func() { upgradeTarget, exitForUpgrade = oldTarget, oldExit })
cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(), EnginesDir: t.TempDir()}
cfg.Superv.WorkDir = t.TempDir()
a, err := New(cfg, nil, nil)
if err != nil {
t.Fatal(err)
}
out := make(chan *llmgwv1.AgentMessage, 1)
go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u1", Version: "v9.9.9", Releases: srv.URL}, out)
select {
case m := <-out:
if r := m.GetUpgrade(); r.GetRequestId() != "u1" || r.GetError() != "" {
t.Fatalf("reply %+v", r)
}
case <-time.After(10 * time.Second):
t.Fatal("no reply")
}
select {
case <-exited:
case <-time.After(5 * time.Second):
t.Fatal("the agent did not restart into the new binary")
}
if b, _ := os.ReadFile(bin); string(b) != "the new agent" {
t.Errorf("binary holds %q", b)
}
// A version with no release answers with why, and does not restart.
go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u2", Version: "v9.9.8", Releases: srv.URL}, out)
if r := (<-out).GetUpgrade(); r.GetError() == "" {
t.Error("an upgrade to a release that does not exist reported success")
}
select {
case <-exited:
t.Error("a failed upgrade restarted the agent")
case <-time.After(time.Second):
}
}