Upgrade the cluster with one command, over the agents' own streams
Every upgrade so far has been a script of mine copying a binary to each machine over ssh, restarting it and polling until it came back — which is the job, but not something the product could do, and which needed ssh into every node. `sudo llmgw ctl upgrade VERSION` on the control plane does it. This machine first: the release for its platform, checked against the release's checksums, the old binary kept beside it, llmgw-server restarted, and a wait until every node that was ready has reconnected. Then each node in turn: the control plane tells its agent the version over the stream it already holds; the agent fetches and checks the release itself, swaps it in with a rename, answers, and exits with status 75 so systemd starts the new binary. Engines keep running, as for any agent restart, and are adopted again. A node is done when it is back on the new version, ready, with as many ready replicas as before; one that is not within the timeout stops the run before the next node, so a bad release reaches one machine. Nodes already on the version are skipped, so running it again resumes. The fetch-and-verify is one package now, shared with onboarding's idea of what a release version looks like. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
3bf4b7b709
commit
116e75f8cb
19 files changed
+1082
-90
No files matched your search
@@ -264,3 +264,22 @@ func TestAGPUFallingOutOfItsDomainTakesTheNodeOut(t *testing.T) {
|
||||
t.Errorf("imex = %+v", n.IMEX)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpgradeWaitsForTheAgentsAnswer(t *testing.T) {
|
||||
s := newTestService()
|
||||
c := &conn{nodeID: "n1", send: make(chan *llmgwv1.ControlMessage, 4), closed: make(chan struct{})}
|
||||
s.conns["n1"] = c
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- s.Upgrade(t.Context(), "n1", "v1.2.3", "https://r") }()
|
||||
|
||||
req := (<-c.send).GetUpgrade()
|
||||
if req.GetVersion() != "v1.2.3" || req.GetReleases() != "https://r" {
|
||||
t.Fatalf("sent %v", req)
|
||||
}
|
||||
s.handle("n1", &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: &llmgwv1.UpgradeReply{
|
||||
RequestId: req.GetRequestId(), Version: "v1.2.3", Error: "checksum mismatch",
|
||||
}}})
|
||||
if err := <-done; err == nil || err.Error() != "checksum mismatch" {
|
||||
t.Errorf("the agent's refusal was not passed on: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -60,6 +60,8 @@ type Service struct {
|
||||
logMu sync.Mutex
|
||||
logWaits map[string]chan *llmgwv1.LogsReply
|
||||
logSeq uint64
|
||||
// upgradeWaits are the upgrade requests awaiting a reply; guarded by logMu.
|
||||
upgradeWaits map[string]chan *llmgwv1.UpgradeReply
|
||||
// scanWaits are the fabric scans awaiting a reply, by request id.
|
||||
scanWaits map[string]chan *llmgwv1.FabricScanReply
|
||||
testWaits map[string]chan *llmgwv1.FabricTestReply
|
||||
@@ -309,6 +311,15 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
|
||||
ch <- m.FabricScan
|
||||
}
|
||||
|
||||
case *llmgwv1.AgentMessage_Upgrade:
|
||||
s.logMu.Lock()
|
||||
ch, ok := s.upgradeWaits[m.Upgrade.GetRequestId()]
|
||||
delete(s.upgradeWaits, m.Upgrade.GetRequestId())
|
||||
s.logMu.Unlock()
|
||||
if ok {
|
||||
ch <- m.Upgrade
|
||||
}
|
||||
|
||||
case *llmgwv1.AgentMessage_Logs:
|
||||
s.logMu.Lock()
|
||||
ch, ok := s.logWaits[m.Logs.GetRequestId()]
|
||||
@@ -842,3 +853,44 @@ func installable(reg *llmgwv1.Register) []string {
|
||||
// The admin API finds the bandwidth test by asserting this; a signature that
|
||||
// drifted would make it quietly nil.
|
||||
var _ fabrictest.Asker = (*Service)(nil)
|
||||
|
||||
// Upgrade tells a node's agent to replace its binary with a release and
|
||||
// restart, and returns once the agent says the new binary is in place — or
|
||||
// why it is not. The agent then exits and comes back on the new version; the
|
||||
// caller watches for that in the inventory.
|
||||
func (s *Service) Upgrade(ctx context.Context, nodeID, version, releases string) error {
|
||||
s.logMu.Lock()
|
||||
if s.upgradeWaits == nil {
|
||||
s.upgradeWaits = map[string]chan *llmgwv1.UpgradeReply{}
|
||||
}
|
||||
s.logSeq++
|
||||
id := fmt.Sprintf("upgrade-%d", s.logSeq)
|
||||
ch := make(chan *llmgwv1.UpgradeReply, 1)
|
||||
s.upgradeWaits[id] = ch
|
||||
s.logMu.Unlock()
|
||||
defer func() {
|
||||
s.logMu.Lock()
|
||||
delete(s.upgradeWaits, id)
|
||||
s.logMu.Unlock()
|
||||
}()
|
||||
|
||||
if err := s.send(nodeID, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_Upgrade{
|
||||
Upgrade: &llmgwv1.UpgradeAgent{RequestId: id, Version: version, Releases: releases},
|
||||
}}); err != nil {
|
||||
return err
|
||||
}
|
||||
s.log.Info("told a node to upgrade", "node", nodeID, "version", version)
|
||||
// A download of some tens of megabytes on a slow link, and no longer: an
|
||||
// agent too old to know this message never answers.
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
select {
|
||||
case r := <-ch:
|
||||
if r.GetError() != "" {
|
||||
return errors.New(r.GetError())
|
||||
}
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("node %s did not answer the upgrade; an agent older than v0.10.4 cannot upgrade itself", nodeID)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user