Ilker ManapandClaude Opus 5.5
116e75f8cb
Every upgrade so far has been a script of mine copying a binary to each machine over ssh, restarting it and polling until it came back — which is the job, but not something the product could do, and which needed ssh into every node. `sudo llmgw ctl upgrade VERSION` on the control plane does it. This machine first: the release for its platform, checked against the release's checksums, the old binary kept beside it, llmgw-server restarted, and a wait until every node that was ready has reconnected. Then each node in turn: the control plane tells its agent the version over the stream it already holds; the agent fetches and checks the release itself, swaps it in with a rename, answers, and exits with status 75 so systemd starts the new binary. Engines keep running, as for any agent restart, and are adopted again. A node is done when it is back on the new version, ready, with as many ready replicas as before; one that is not within the timeout stops the run before the next node, so a bad release reaches one machine. Nodes already on the version are skipped, so running it again resumes. The fetch-and-verify is one package now, shared with onboarding's idea of what a release version looks like. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
89 lines
2.7 KiB
Go
89 lines
2.7 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
)
|
|
|
|
// fakeFleet is an admin API whose nodes come back on the version they are told
|
|
// to upgrade to — except those listed in stuck.
|
|
type fakeFleet struct {
|
|
mu sync.Mutex
|
|
versions map[string]string
|
|
stuck map[string]bool
|
|
told []string
|
|
}
|
|
|
|
func (f *fakeFleet) server(t *testing.T) *httptest.Server {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
switch {
|
|
case r.URL.Path == "/api/v1/nodes":
|
|
var out []adminapi.NodeView
|
|
for id, v := range f.versions {
|
|
out = append(out, adminapi.NodeView{ID: id, State: "ready", AgentVersion: v})
|
|
}
|
|
_ = json.NewEncoder(w).Encode(out)
|
|
case r.URL.Path == "/api/v1/replicas":
|
|
var out []adminapi.ReplicaView
|
|
for id := range f.versions {
|
|
out = append(out, adminapi.ReplicaView{ID: "r-" + id, State: "ready",
|
|
Shards: []adminapi.ShardView{{Node: id}}})
|
|
}
|
|
_ = json.NewEncoder(w).Encode(out)
|
|
case strings.HasSuffix(r.URL.Path, "/upgrade"):
|
|
id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/nodes/"), "/upgrade")
|
|
var req adminapi.UpgradeRequest
|
|
_ = json.NewDecoder(r.Body).Decode(&req)
|
|
f.told = append(f.told, id)
|
|
if !f.stuck[id] {
|
|
f.versions[id] = req.Version
|
|
}
|
|
_, _ = w.Write([]byte(`{}`))
|
|
default:
|
|
http.NotFound(w, r)
|
|
}
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
return srv
|
|
}
|
|
|
|
func TestAnUpgradeGoesNodeByNodeAndSkipsTheDone(t *testing.T) {
|
|
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.1", "c": "v1.0.0"}}
|
|
srv := f.server(t)
|
|
var out bytes.Buffer
|
|
err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 5*time.Second)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Join(f.told, ",") != "a,c" {
|
|
t.Errorf("told %v; b was already on the version", f.told)
|
|
}
|
|
if !strings.Contains(out.String(), "2 node(s) upgraded to v1.0.1, 1 already on it") {
|
|
t.Errorf("output:\n%s", out.String())
|
|
}
|
|
}
|
|
|
|
func TestAnUpgradeStopsAtTheFirstNodeThatDoesNotComeBack(t *testing.T) {
|
|
// A bad release reaches one node, not all of them.
|
|
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.0"}, stuck: map[string]bool{"a": true}}
|
|
srv := f.server(t)
|
|
var out bytes.Buffer
|
|
err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 3*time.Second)
|
|
if err == nil || !strings.Contains(err.Error(), "a did not come back") {
|
|
t.Fatalf("err %v", err)
|
|
}
|
|
if len(f.told) != 1 {
|
|
t.Errorf("went on to %v after a node failed to come back", f.told)
|
|
}
|
|
}
|