Cluster upgrades: nodes told together with progress said, done when back on the version, the download kept past a broken stream
CI / test (push) Successful in 10m2s
CI / test (push) Successful in 10m2s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d6f8f433f4
commit
12464ce0f6
10 files changed
+244
-35
No files matched your search
@@ -47,7 +47,9 @@ import (
|
||||
|
||||
// Agent runs on a GPU node.
|
||||
type Agent struct {
|
||||
cfg Config
|
||||
// upgrading is true while an upgrade downloads.
|
||||
upgrading atomic.Bool
|
||||
cfg Config
|
||||
// answered says the current session heard from its control plane, which
|
||||
// starts the reconnect backoff again when it ends.
|
||||
answered atomic.Bool
|
||||
|
||||
@@ -35,12 +35,25 @@ func (a *Agent) upgrade(ctx context.Context, m *llmgwv1.UpgradeAgent, out chan<-
|
||||
select {
|
||||
case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: reply}}:
|
||||
case <-ctx.Done():
|
||||
case <-time.After(5 * time.Second):
|
||||
}
|
||||
}
|
||||
// One at a time: a second request while one downloads would race it to
|
||||
// the same file.
|
||||
if !a.upgrading.CompareAndSwap(false, true) {
|
||||
reply.Error = "an upgrade is already under way on this node"
|
||||
send()
|
||||
return
|
||||
}
|
||||
defer a.upgrading.Store(false)
|
||||
a.log.Info("upgrading", "to", m.GetVersion(), "from", a.cfg.Version)
|
||||
self, err := upgradeTarget()
|
||||
if err == nil {
|
||||
dctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
// Not the stream's context: a stream that breaks while the release
|
||||
// downloads — a control plane restarting, leadership moving — cancelled
|
||||
// the upgrade with nobody told. The node registering again on the
|
||||
// version is what the control plane watches for.
|
||||
dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), upgradeDownload)
|
||||
_, err = release.Install(dctx, nil, m.GetReleases(), m.GetVersion(), runtime.GOOS, runtime.GOARCH, self)
|
||||
cancel()
|
||||
}
|
||||
@@ -53,6 +66,13 @@ func (a *Agent) upgrade(ctx context.Context, m *llmgwv1.UpgradeAgent, out chan<-
|
||||
send()
|
||||
a.log.Info("upgraded; restarting into the new version", "to", m.GetVersion())
|
||||
// A moment for the reply to leave before the stream goes.
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
time.Sleep(upgradeLinger)
|
||||
exitForUpgrade()
|
||||
}
|
||||
|
||||
// upgradeDownload bounds an upgrade's download; the control plane waits
|
||||
// longer (agentsvc.UpgradeWait). upgradeLinger is the moment its answer has
|
||||
// to leave before the agent exits; a variable so tests need not wait it.
|
||||
const upgradeDownload = 10 * time.Minute
|
||||
|
||||
var upgradeLinger = 2 * time.Second
|
||||
Reference in new issue
Block a user