Ilker ManapandClaude Opus 5.5
12464ce0f6
CI / test (push) Successful in 10m2s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
79 lines
2.8 KiB
Go
79 lines
2.8 KiB
Go
package agent
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"runtime"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
)
|
|
|
|
// upgradeExit is the exit status after an upgrade: non-zero, so systemd's
|
|
// Restart=on-failure starts the new binary, and distinct, so the journal says
|
|
// why the agent went.
|
|
const upgradeExit = 75
|
|
|
|
// exitForUpgrade ends the process; a variable so a test can see it called
|
|
// rather than have its own process end.
|
|
var exitForUpgrade = func() { os.Exit(upgradeExit) }
|
|
|
|
// upgradeTarget is the binary an upgrade replaces; a variable so a test
|
|
// replaces a file of its own rather than the test binary running it.
|
|
var upgradeTarget = os.Executable
|
|
|
|
// upgrade replaces this agent's binary with a release and restarts into it.
|
|
//
|
|
// Engines are not touched: an agent restart leaves them running and adopts
|
|
// them again, which is what makes upgrading a node something that can be done
|
|
// while it serves.
|
|
func (a *Agent) upgrade(ctx context.Context, m *llmgwv1.UpgradeAgent, out chan<- *llmgwv1.AgentMessage) {
|
|
reply := &llmgwv1.UpgradeReply{RequestId: m.GetRequestId(), Version: m.GetVersion()}
|
|
send := func() {
|
|
select {
|
|
case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: reply}}:
|
|
case <-ctx.Done():
|
|
case <-time.After(5 * time.Second):
|
|
}
|
|
}
|
|
// One at a time: a second request while one downloads would race it to
|
|
// the same file.
|
|
if !a.upgrading.CompareAndSwap(false, true) {
|
|
reply.Error = "an upgrade is already under way on this node"
|
|
send()
|
|
return
|
|
}
|
|
defer a.upgrading.Store(false)
|
|
a.log.Info("upgrading", "to", m.GetVersion(), "from", a.cfg.Version)
|
|
self, err := upgradeTarget()
|
|
if err == nil {
|
|
// Not the stream's context: a stream that breaks while the release
|
|
// downloads — a control plane restarting, leadership moving — cancelled
|
|
// the upgrade with nobody told. The node registering again on the
|
|
// version is what the control plane watches for.
|
|
dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), upgradeDownload)
|
|
_, err = release.Install(dctx, nil, m.GetReleases(), m.GetVersion(), runtime.GOOS, runtime.GOARCH, self)
|
|
cancel()
|
|
}
|
|
if err != nil {
|
|
reply.Error = err.Error()
|
|
a.log.Error("upgrade failed; still running the old version", "to", m.GetVersion(), "err", err)
|
|
send()
|
|
return
|
|
}
|
|
send()
|
|
a.log.Info("upgraded; restarting into the new version", "to", m.GetVersion())
|
|
// A moment for the reply to leave before the stream goes.
|
|
time.Sleep(upgradeLinger)
|
|
exitForUpgrade()
|
|
}
|
|
|
|
// upgradeDownload bounds an upgrade's download; the control plane waits
|
|
// longer (agentsvc.UpgradeWait). upgradeLinger is the moment its answer has
|
|
// to leave before the agent exits; a variable so tests need not wait it.
|
|
const upgradeDownload = 10 * time.Minute
|
|
|
|
var upgradeLinger = 2 * time.Second
|