Cluster upgrades: nodes told together with progress said, done when back on the version, the download kept past a broken stream
CI / test (push) Successful in 10m2s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-30 11:14:02 +01:00
1 parent d6f8f433f4
commit 12464ce0f6
10 files changed
+244 -35

No files matched your search

+98 -19
View File
@@ -11,11 +11,13 @@ import (
"runtime"
"slices"
"strings"
"sync"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
@@ -244,6 +246,7 @@ func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient,
}
slices.SortFunc(nodes, func(a, b adminapi.NodeView) int { return strings.Compare(a.ID, b.ID) })
var todo []adminapi.NodeView
var away []string
skipped := 0
for _, n := range nodes {
if len(only) > 0 && !slices.Contains(only, n.ID) {
@@ -253,8 +256,17 @@ func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient,
skipped++
continue
}
// A node that is not connected cannot be told, and one tried first
// stopped every run at it: passed over, and named at the end.
if n.State == "lost" || n.State == "unreachable" {
away = append(away, n.ID)
continue
}
todo = append(todo, n)
}
if len(away) > 0 {
fmt.Fprintf(out, "Passed over, not connected: %s\n", strings.Join(away, ", "))
}
parallel = max(parallel, 1)
start, size, done := time.Now(), 1, 0
for len(todo) > 0 {
@@ -296,7 +308,10 @@ func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient,
}
// upgradeWave tells every node in the wave at once, and waits until each is
// back on v, ready, with as many ready replicas as it had.
// back on v and serving what it served.
//
// Told together, not one after another: each answer can take minutes, and a
// wave of twenty waited for them in turn with nothing on the screen.
func upgradeWave(ctx context.Context, out io.Writer, client func() *ctlClient, v string,
wave []adminapi.NodeView, perNode time.Duration,
) error {
@@ -308,35 +323,61 @@ func upgradeWave(ctx context.Context, out io.Writer, client func() *ctlClient, v
}
fmt.Fprintf(out, "== %d node(s): %s\n", len(wave), abbreviate(ids, 8))
body, _ := json.Marshal(adminapi.UpgradeRequest{Version: v})
var failed []string
var (
mu sync.Mutex
failed []string
wg sync.WaitGroup
)
for _, n := range wave {
// The control plane waits for the agent to fetch and check the
// release, up to five minutes; fifteen seconds gave up on a node
// that was upgrading.
if err := client().patient(6*time.Minute).post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil {
failed = append(failed, fmt.Sprintf("%s: %v", n.ID, err))
}
wg.Add(1)
go func() {
defer wg.Done()
// The control plane waits for the agent's answer, or for it to
// come back on v, longer than the agent gives its download.
err := client().patient(agentsvc.UpgradeWait+time.Minute).post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil)
mu.Lock()
defer mu.Unlock()
if err != nil {
failed = append(failed, fmt.Sprintf("%s: %v", n.ID, err))
fmt.Fprintf(out, " %s: not upgraded: %v\n", n.ID, err)
return
}
fmt.Fprintf(out, " %s: on %s, coming back\n", n.ID, v)
}()
}
if len(failed) > 0 {
return fmt.Errorf("not told to upgrade:\n %s\n`llmgw ctl logs NODE` says more", strings.Join(failed, "\n "))
}
waiting := slices.Clone(ids)
wg.Wait()
waiting := slices.DeleteFunc(slices.Clone(ids), func(id string) bool {
return slices.ContainsFunc(failed, func(f string) bool { return strings.HasPrefix(f, id+": ") })
})
deadline := time.Now().Add(perNode)
for {
backAt := map[string]time.Time{}
said := time.Now()
for len(waiting) > 0 {
var now []adminapi.NodeView
_ = client().get(ctx, "/api/v1/nodes", &now)
after := readyReplicasByNode(ctx, client)
var reps []adminapi.ReplicaView
_ = client().get(ctx, "/api/v1/replicas", &reps)
after, starting := replicasByNode(reps)
state := map[string]adminapi.NodeView{}
for _, x := range now {
state[x.ID] = x
}
waiting = slices.DeleteFunc(waiting, func(id string) bool {
cur := state[id]
return cur.AgentVersion == v && cur.State == "ready" && after[id] >= before[id]
// Connected and on v: ready, or cordoned or draining as it
// was, which an upgrade does not undo.
if cur.AgentVersion != v || (cur.State != "ready" && cur.State != "cordoned" && cur.State != "draining") {
return false
}
if _, ok := backAt[id]; !ok {
backAt[id] = time.Now()
}
// Serving what it served; or, a while on, nothing on it still
// starting: a model that went idle meanwhile is not one lost.
return after[id] >= before[id] || (time.Since(backAt[id]) > 90*time.Second && starting[id] == 0)
})
if len(waiting) == 0 {
fmt.Fprintf(out, " done in %s\n", time.Since(start).Round(time.Second))
return nil
break
}
if time.Now().After(deadline) {
var why []string
@@ -345,11 +386,49 @@ func upgradeWave(ctx context.Context, out io.Writer, client func() *ctlClient, v
why = append(why, fmt.Sprintf("%s (version %q, state %q, replicas %d/%d)",
id, cur.AgentVersion, cur.State, after[id], before[id]))
}
return fmt.Errorf("not back within %s:\n %s\n`llmgw ctl logs NODE` says why, and each keeps its "+
"previous binary beside the new one", perNode, strings.Join(why, "\n "))
failed = append(failed, fmt.Sprintf("not back within %s: %s", perNode, strings.Join(why, ", ")))
break
}
if time.Since(said) >= 30*time.Second {
var w []string
for _, id := range waiting {
cur := state[id]
w = append(w, fmt.Sprintf("%s (%s, %s, replicas %d/%d)", id, orUnknown(cur.AgentVersion), cur.State,
after[id], before[id]))
}
fmt.Fprintf(out, " waiting for %s\n", abbreviate(w, 8))
said = time.Now()
}
time.Sleep(2 * time.Second)
}
if len(failed) > 0 {
return fmt.Errorf("not upgraded:\n %s\n`llmgw ctl logs NODE` says why; each keeps its previous binary beside the new one",
strings.Join(failed, "\n "))
}
fmt.Fprintf(out, " done in %s\n", time.Since(start).Round(time.Second))
return nil
}
// replicasByNode counts each node's ready replicas, and those still placing
// or starting.
func replicasByNode(reps []adminapi.ReplicaView) (ready, starting map[string]int) {
ready, starting = map[string]int{}, map[string]int{}
for _, r := range reps {
seen := map[string]bool{}
for _, sh := range r.Shards {
if seen[sh.Node] {
continue
}
seen[sh.Node] = true
switch r.State {
case "ready":
ready[sh.Node]++
case "placing", "starting":
starting[sh.Node]++
}
}
}
return ready, starting
}
// abbreviate lists up to n names, and says how many more.