Cluster upgrades: nodes told together with progress said, done when back on the version, the download kept past a broken stream
CI / test (push) Successful in 10m2s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-30 11:14:02 +01:00
1 parent d6f8f433f4
commit 12464ce0f6
10 files changed
+244 -35

No files matched your search

@@ -3,6 +3,7 @@ package agentsvc
import (
"log/slog"
"testing"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
@@ -284,6 +285,28 @@ func TestAnUpgradeWaitsForTheAgentsAnswer(t *testing.T) {
}
}
// The agent's answer lost — it exits a moment after sending it — and the node
// back on the version: the upgrade is done, not five minutes of waiting and a
// failure blaming an old agent.
func TestAnUpgradeIsDoneWhenTheNodeComesBackOnTheVersion(t *testing.T) {
s := newTestService()
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready, AgentVersion: "v1.2.2"})
c := &conn{nodeID: "n1", send: make(chan *llmgwv1.ControlMessage, 4), closed: make(chan struct{})}
s.conns["n1"] = c
done := make(chan error, 1)
go func() { done <- s.Upgrade(t.Context(), "n1", "v1.2.3", "https://r") }()
<-c.send
nodeWith(s, inventory.Node{ID: "n1", State: inventory.Ready, AgentVersion: "v1.2.3"})
select {
case err := <-done:
if err != nil {
t.Fatal(err)
}
case <-time.After(10 * time.Second):
t.Fatal("still waiting for an answer the node will never send")
}
}
func TestAnotherTenantOnACardIsJudgedEveryHeartbeat(t *testing.T) {
// Found on cuda01: ComfyUI held 5 GB when the agent started, so the card
// was kept out of placement. ComfyUI let go of it and the card stayed
+34 -10
View File
@@ -1158,21 +1158,45 @@ func (s *Service) Upgrade(ctx context.Context, nodeID, version, releases string)
return err
}
s.log.Info("told a node to upgrade", "node", nodeID, "version", version)
// A download of some tens of megabytes on a slow link, and no longer: an
// agent too old to know this message never answers.
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
// Longer than the agent gives its download, so its own answer — done, or
// why not — is what is heard. Its answer can still be lost: it exits a
// moment after sending it, and a stream that breaks takes it along. The
// node registering again on the version says the same, and is watched for.
ctx, cancel := context.WithTimeout(ctx, UpgradeWait)
defer cancel()
select {
case r := <-ch:
if r.GetError() != "" {
return errors.New(r.GetError())
tick := time.NewTicker(2 * time.Second)
defer tick.Stop()
for {
select {
case r := <-ch:
if r.GetError() != "" {
return errors.New(r.GetError())
}
return nil
case <-tick.C:
if n, ok := s.inv.Get(nodeID); ok && n.AgentVersion == version {
return nil
}
case <-ctx.Done():
n, ok := s.inv.Get(nodeID)
switch {
case !slices.Contains(s.ConnectedNodes(), nodeID):
return fmt.Errorf("node %s went away during the upgrade and has not come back: on it, "+
"`systemctl status llmgw-agent` and `journalctl -u llmgw-agent` say why", nodeID)
case ok && n.AgentVersion != "":
return fmt.Errorf("node %s did not answer the upgrade and still runs %s: its download is slow or stuck, "+
"or its agent is older than v0.10.4, which cannot upgrade itself", nodeID, n.AgentVersion)
default:
return fmt.Errorf("node %s did not answer the upgrade; an agent older than v0.10.4 cannot upgrade itself", nodeID)
}
}
return nil
case <-ctx.Done():
return fmt.Errorf("node %s did not answer the upgrade; an agent older than v0.10.4 cannot upgrade itself", nodeID)
}
}
// UpgradeWait is how long the control plane waits for an agent to upgrade:
// longer than the agent gives its download (agent.upgradeDownload).
const UpgradeWait = 11 * time.Minute
// wearFrom is a card's wear, when the agent read it.
func wearFrom(g *llmgwv1.GPUSensor) *inventory.GPUWear {
if !g.GetHasWear() {