Cluster upgrades: nodes told together with progress said, done when back on the version, the download kept past a broken stream
CI / test (push) Successful in 10m2s
CI / test (push) Successful in 10m2s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d6f8f433f4
commit
12464ce0f6
10 files changed
+244
-35
No files matched your search
+98
-19
@@ -11,11 +11,13 @@ import (
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
@@ -244,6 +246,7 @@ func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient,
|
||||
}
|
||||
slices.SortFunc(nodes, func(a, b adminapi.NodeView) int { return strings.Compare(a.ID, b.ID) })
|
||||
var todo []adminapi.NodeView
|
||||
var away []string
|
||||
skipped := 0
|
||||
for _, n := range nodes {
|
||||
if len(only) > 0 && !slices.Contains(only, n.ID) {
|
||||
@@ -253,8 +256,17 @@ func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient,
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
// A node that is not connected cannot be told, and one tried first
|
||||
// stopped every run at it: passed over, and named at the end.
|
||||
if n.State == "lost" || n.State == "unreachable" {
|
||||
away = append(away, n.ID)
|
||||
continue
|
||||
}
|
||||
todo = append(todo, n)
|
||||
}
|
||||
if len(away) > 0 {
|
||||
fmt.Fprintf(out, "Passed over, not connected: %s\n", strings.Join(away, ", "))
|
||||
}
|
||||
parallel = max(parallel, 1)
|
||||
start, size, done := time.Now(), 1, 0
|
||||
for len(todo) > 0 {
|
||||
@@ -296,7 +308,10 @@ func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient,
|
||||
}
|
||||
|
||||
// upgradeWave tells every node in the wave at once, and waits until each is
|
||||
// back on v, ready, with as many ready replicas as it had.
|
||||
// back on v and serving what it served.
|
||||
//
|
||||
// Told together, not one after another: each answer can take minutes, and a
|
||||
// wave of twenty waited for them in turn with nothing on the screen.
|
||||
func upgradeWave(ctx context.Context, out io.Writer, client func() *ctlClient, v string,
|
||||
wave []adminapi.NodeView, perNode time.Duration,
|
||||
) error {
|
||||
@@ -308,35 +323,61 @@ func upgradeWave(ctx context.Context, out io.Writer, client func() *ctlClient, v
|
||||
}
|
||||
fmt.Fprintf(out, "== %d node(s): %s\n", len(wave), abbreviate(ids, 8))
|
||||
body, _ := json.Marshal(adminapi.UpgradeRequest{Version: v})
|
||||
var failed []string
|
||||
var (
|
||||
mu sync.Mutex
|
||||
failed []string
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
for _, n := range wave {
|
||||
// The control plane waits for the agent to fetch and check the
|
||||
// release, up to five minutes; fifteen seconds gave up on a node
|
||||
// that was upgrading.
|
||||
if err := client().patient(6*time.Minute).post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil {
|
||||
failed = append(failed, fmt.Sprintf("%s: %v", n.ID, err))
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
// The control plane waits for the agent's answer, or for it to
|
||||
// come back on v, longer than the agent gives its download.
|
||||
err := client().patient(agentsvc.UpgradeWait+time.Minute).post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if err != nil {
|
||||
failed = append(failed, fmt.Sprintf("%s: %v", n.ID, err))
|
||||
fmt.Fprintf(out, " %s: not upgraded: %v\n", n.ID, err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(out, " %s: on %s, coming back\n", n.ID, v)
|
||||
}()
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("not told to upgrade:\n %s\n`llmgw ctl logs NODE` says more", strings.Join(failed, "\n "))
|
||||
}
|
||||
waiting := slices.Clone(ids)
|
||||
wg.Wait()
|
||||
waiting := slices.DeleteFunc(slices.Clone(ids), func(id string) bool {
|
||||
return slices.ContainsFunc(failed, func(f string) bool { return strings.HasPrefix(f, id+": ") })
|
||||
})
|
||||
deadline := time.Now().Add(perNode)
|
||||
for {
|
||||
backAt := map[string]time.Time{}
|
||||
said := time.Now()
|
||||
for len(waiting) > 0 {
|
||||
var now []adminapi.NodeView
|
||||
_ = client().get(ctx, "/api/v1/nodes", &now)
|
||||
after := readyReplicasByNode(ctx, client)
|
||||
var reps []adminapi.ReplicaView
|
||||
_ = client().get(ctx, "/api/v1/replicas", &reps)
|
||||
after, starting := replicasByNode(reps)
|
||||
state := map[string]adminapi.NodeView{}
|
||||
for _, x := range now {
|
||||
state[x.ID] = x
|
||||
}
|
||||
waiting = slices.DeleteFunc(waiting, func(id string) bool {
|
||||
cur := state[id]
|
||||
return cur.AgentVersion == v && cur.State == "ready" && after[id] >= before[id]
|
||||
// Connected and on v: ready, or cordoned or draining as it
|
||||
// was, which an upgrade does not undo.
|
||||
if cur.AgentVersion != v || (cur.State != "ready" && cur.State != "cordoned" && cur.State != "draining") {
|
||||
return false
|
||||
}
|
||||
if _, ok := backAt[id]; !ok {
|
||||
backAt[id] = time.Now()
|
||||
}
|
||||
// Serving what it served; or, a while on, nothing on it still
|
||||
// starting: a model that went idle meanwhile is not one lost.
|
||||
return after[id] >= before[id] || (time.Since(backAt[id]) > 90*time.Second && starting[id] == 0)
|
||||
})
|
||||
if len(waiting) == 0 {
|
||||
fmt.Fprintf(out, " done in %s\n", time.Since(start).Round(time.Second))
|
||||
return nil
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
var why []string
|
||||
@@ -345,11 +386,49 @@ func upgradeWave(ctx context.Context, out io.Writer, client func() *ctlClient, v
|
||||
why = append(why, fmt.Sprintf("%s (version %q, state %q, replicas %d/%d)",
|
||||
id, cur.AgentVersion, cur.State, after[id], before[id]))
|
||||
}
|
||||
return fmt.Errorf("not back within %s:\n %s\n`llmgw ctl logs NODE` says why, and each keeps its "+
|
||||
"previous binary beside the new one", perNode, strings.Join(why, "\n "))
|
||||
failed = append(failed, fmt.Sprintf("not back within %s: %s", perNode, strings.Join(why, ", ")))
|
||||
break
|
||||
}
|
||||
if time.Since(said) >= 30*time.Second {
|
||||
var w []string
|
||||
for _, id := range waiting {
|
||||
cur := state[id]
|
||||
w = append(w, fmt.Sprintf("%s (%s, %s, replicas %d/%d)", id, orUnknown(cur.AgentVersion), cur.State,
|
||||
after[id], before[id]))
|
||||
}
|
||||
fmt.Fprintf(out, " waiting for %s\n", abbreviate(w, 8))
|
||||
said = time.Now()
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("not upgraded:\n %s\n`llmgw ctl logs NODE` says why; each keeps its previous binary beside the new one",
|
||||
strings.Join(failed, "\n "))
|
||||
}
|
||||
fmt.Fprintf(out, " done in %s\n", time.Since(start).Round(time.Second))
|
||||
return nil
|
||||
}
|
||||
|
||||
// replicasByNode counts each node's ready replicas, and those still placing
|
||||
// or starting.
|
||||
func replicasByNode(reps []adminapi.ReplicaView) (ready, starting map[string]int) {
|
||||
ready, starting = map[string]int{}, map[string]int{}
|
||||
for _, r := range reps {
|
||||
seen := map[string]bool{}
|
||||
for _, sh := range r.Shards {
|
||||
if seen[sh.Node] {
|
||||
continue
|
||||
}
|
||||
seen[sh.Node] = true
|
||||
switch r.State {
|
||||
case "ready":
|
||||
ready[sh.Node]++
|
||||
case "placing", "starting":
|
||||
starting[sh.Node]++
|
||||
}
|
||||
}
|
||||
}
|
||||
return ready, starting
|
||||
}
|
||||
|
||||
// abbreviate lists up to n names, and says how many more.
|
||||
|
||||
Reference in new issue
Block a user