Cluster upgrades: nodes told together with progress said, done when back on the version, the download kept past a broken stream
CI / test (push) Successful in 10m2s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-30 11:14:02 +01:00
1 parent d6f8f433f4
commit 12464ce0f6
10 files changed
+244 -35

No files matched your search

+3 -1
View File
@@ -47,7 +47,9 @@ import (
// Agent runs on a GPU node.
type Agent struct {
cfg Config
// upgrading is true while an upgrade downloads.
upgrading atomic.Bool
cfg Config
// answered says the current session heard from its control plane, which
// starts the reconnect backoff again when it ends.
answered atomic.Bool
+22 -2
View File
@@ -35,12 +35,25 @@ func (a *Agent) upgrade(ctx context.Context, m *llmgwv1.UpgradeAgent, out chan<-
select {
case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: reply}}:
case <-ctx.Done():
case <-time.After(5 * time.Second):
}
}
// One at a time: a second request while one downloads would race it to
// the same file.
if !a.upgrading.CompareAndSwap(false, true) {
reply.Error = "an upgrade is already under way on this node"
send()
return
}
defer a.upgrading.Store(false)
a.log.Info("upgrading", "to", m.GetVersion(), "from", a.cfg.Version)
self, err := upgradeTarget()
if err == nil {
dctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
// Not the stream's context: a stream that breaks while the release
// downloads — a control plane restarting, leadership moving — cancelled
// the upgrade with nobody told. The node registering again on the
// version is what the control plane watches for.
dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), upgradeDownload)
_, err = release.Install(dctx, nil, m.GetReleases(), m.GetVersion(), runtime.GOOS, runtime.GOARCH, self)
cancel()
}
@@ -53,6 +66,13 @@ func (a *Agent) upgrade(ctx context.Context, m *llmgwv1.UpgradeAgent, out chan<-
send()
a.log.Info("upgraded; restarting into the new version", "to", m.GetVersion())
// A moment for the reply to leave before the stream goes.
time.Sleep(500 * time.Millisecond)
time.Sleep(upgradeLinger)
exitForUpgrade()
}
// upgradeDownload bounds an upgrade's download; the control plane waits
// longer (agentsvc.UpgradeWait). upgradeLinger is the moment its answer has
// to leave before the agent exits; a variable so tests need not wait it.
const upgradeDownload = 10 * time.Minute
var upgradeLinger = 2 * time.Second