Upgrade the cluster with one command, over the agents' own streams
Every upgrade so far has been a script of mine copying a binary to each machine over ssh, restarting it and polling until it came back — which is the job, but not something the product could do, and which needed ssh into every node. `sudo llmgw ctl upgrade VERSION` on the control plane does it. This machine first: the release for its platform, checked against the release's checksums, the old binary kept beside it, llmgw-server restarted, and a wait until every node that was ready has reconnected. Then each node in turn: the control plane tells its agent the version over the stream it already holds; the agent fetches and checks the release itself, swaps it in with a rename, answers, and exits with status 75 so systemd starts the new binary. Engines keep running, as for any agent restart, and are adopted again. A node is done when it is back on the new version, ready, with as many ready replicas as before; one that is not within the timeout stops the run before the next node, so a bad release reaches one machine. Nodes already on the version are skipped, so running it again resumes. The fetch-and-verify is one package now, shared with onboarding's idea of what a release version looks like. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
3bf4b7b709
commit
116e75f8cb
19 files changed
+1082
-90
No files matched your search
@@ -0,0 +1,88 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// fakeFleet is an admin API whose nodes come back on the version they are told
|
||||
// to upgrade to — except those listed in stuck.
|
||||
type fakeFleet struct {
|
||||
mu sync.Mutex
|
||||
versions map[string]string
|
||||
stuck map[string]bool
|
||||
told []string
|
||||
}
|
||||
|
||||
func (f *fakeFleet) server(t *testing.T) *httptest.Server {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
switch {
|
||||
case r.URL.Path == "/api/v1/nodes":
|
||||
var out []adminapi.NodeView
|
||||
for id, v := range f.versions {
|
||||
out = append(out, adminapi.NodeView{ID: id, State: "ready", AgentVersion: v})
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(out)
|
||||
case r.URL.Path == "/api/v1/replicas":
|
||||
var out []adminapi.ReplicaView
|
||||
for id := range f.versions {
|
||||
out = append(out, adminapi.ReplicaView{ID: "r-" + id, State: "ready",
|
||||
Shards: []adminapi.ShardView{{Node: id}}})
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(out)
|
||||
case strings.HasSuffix(r.URL.Path, "/upgrade"):
|
||||
id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/nodes/"), "/upgrade")
|
||||
var req adminapi.UpgradeRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
f.told = append(f.told, id)
|
||||
if !f.stuck[id] {
|
||||
f.versions[id] = req.Version
|
||||
}
|
||||
_, _ = w.Write([]byte(`{}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestAnUpgradeGoesNodeByNodeAndSkipsTheDone(t *testing.T) {
|
||||
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.1", "c": "v1.0.0"}}
|
||||
srv := f.server(t)
|
||||
var out bytes.Buffer
|
||||
err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(f.told, ",") != "a,c" {
|
||||
t.Errorf("told %v; b was already on the version", f.told)
|
||||
}
|
||||
if !strings.Contains(out.String(), "2 node(s) upgraded to v1.0.1, 1 already on it") {
|
||||
t.Errorf("output:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpgradeStopsAtTheFirstNodeThatDoesNotComeBack(t *testing.T) {
|
||||
// A bad release reaches one node, not all of them.
|
||||
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.0"}, stuck: map[string]bool{"a": true}}
|
||||
srv := f.server(t)
|
||||
var out bytes.Buffer
|
||||
err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 3*time.Second)
|
||||
if err == nil || !strings.Contains(err.Error(), "a did not come back") {
|
||||
t.Fatalf("err %v", err)
|
||||
}
|
||||
if len(f.told) != 1 {
|
||||
t.Errorf("went on to %v after a node failed to come back", f.told)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user