package agent import ( "context" "os" "runtime" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/release" llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" ) // upgradeExit is the exit status after an upgrade: non-zero, so systemd's // Restart=on-failure starts the new binary, and distinct, so the journal says // why the agent went. const upgradeExit = 75 // exitForUpgrade ends the process; a variable so a test can see it called // rather than have its own process end. var exitForUpgrade = func() { os.Exit(upgradeExit) } // upgradeTarget is the binary an upgrade replaces; a variable so a test // replaces a file of its own rather than the test binary running it. var upgradeTarget = os.Executable // upgrade replaces this agent's binary with a release and restarts into it. // // Engines are not touched: an agent restart leaves them running and adopts // them again, which is what makes upgrading a node something that can be done // while it serves. func (a *Agent) upgrade(ctx context.Context, m *llmgwv1.UpgradeAgent, out chan<- *llmgwv1.AgentMessage) { reply := &llmgwv1.UpgradeReply{RequestId: m.GetRequestId(), Version: m.GetVersion()} send := func() { select { case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: reply}}: case <-ctx.Done(): case <-time.After(5 * time.Second): } } // One at a time: a second request while one downloads would race it to // the same file. if !a.upgrading.CompareAndSwap(false, true) { reply.Error = "an upgrade is already under way on this node" send() return } defer a.upgrading.Store(false) a.log.Info("upgrading", "to", m.GetVersion(), "from", a.cfg.Version) self, err := upgradeTarget() if err == nil { // Not the stream's context: a stream that breaks while the release // downloads — a control plane restarting, leadership moving — cancelled // the upgrade with nobody told. The node registering again on the // version is what the control plane watches for. dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), upgradeDownload) _, err = release.Install(dctx, nil, m.GetReleases(), m.GetVersion(), runtime.GOOS, runtime.GOARCH, self) cancel() } if err != nil { reply.Error = err.Error() a.log.Error("upgrade failed; still running the old version", "to", m.GetVersion(), "err", err) send() return } send() a.log.Info("upgraded; restarting into the new version", "to", m.GetVersion()) // A moment for the reply to leave before the stream goes. time.Sleep(upgradeLinger) exitForUpgrade() } // upgradeDownload bounds an upgrade's download; the control plane waits // longer (agentsvc.UpgradeWait). upgradeLinger is the moment its answer has // to leave before the agent exits; a variable so tests need not wait it. const upgradeDownload = 10 * time.Minute var upgradeLinger = 2 * time.Second