Cluster upgrades: nodes told together with progress said, done when back on the version, the download kept past a broken stream
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d6f8f433f4
commit
12464ce0f6
10 files changed
+244
-35
No files matched your search
+98
-19
@@ -11,11 +11,13 @@ import (
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
@@ -244,6 +246,7 @@ func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient,
|
||||
}
|
||||
slices.SortFunc(nodes, func(a, b adminapi.NodeView) int { return strings.Compare(a.ID, b.ID) })
|
||||
var todo []adminapi.NodeView
|
||||
var away []string
|
||||
skipped := 0
|
||||
for _, n := range nodes {
|
||||
if len(only) > 0 && !slices.Contains(only, n.ID) {
|
||||
@@ -253,8 +256,17 @@ func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient,
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
// A node that is not connected cannot be told, and one tried first
|
||||
// stopped every run at it: passed over, and named at the end.
|
||||
if n.State == "lost" || n.State == "unreachable" {
|
||||
away = append(away, n.ID)
|
||||
continue
|
||||
}
|
||||
todo = append(todo, n)
|
||||
}
|
||||
if len(away) > 0 {
|
||||
fmt.Fprintf(out, "Passed over, not connected: %s\n", strings.Join(away, ", "))
|
||||
}
|
||||
parallel = max(parallel, 1)
|
||||
start, size, done := time.Now(), 1, 0
|
||||
for len(todo) > 0 {
|
||||
@@ -296,7 +308,10 @@ func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient,
|
||||
}
|
||||
|
||||
// upgradeWave tells every node in the wave at once, and waits until each is
|
||||
// back on v, ready, with as many ready replicas as it had.
|
||||
// back on v and serving what it served.
|
||||
//
|
||||
// Told together, not one after another: each answer can take minutes, and a
|
||||
// wave of twenty waited for them in turn with nothing on the screen.
|
||||
func upgradeWave(ctx context.Context, out io.Writer, client func() *ctlClient, v string,
|
||||
wave []adminapi.NodeView, perNode time.Duration,
|
||||
) error {
|
||||
@@ -308,35 +323,61 @@ func upgradeWave(ctx context.Context, out io.Writer, client func() *ctlClient, v
|
||||
}
|
||||
fmt.Fprintf(out, "== %d node(s): %s\n", len(wave), abbreviate(ids, 8))
|
||||
body, _ := json.Marshal(adminapi.UpgradeRequest{Version: v})
|
||||
var failed []string
|
||||
var (
|
||||
mu sync.Mutex
|
||||
failed []string
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
for _, n := range wave {
|
||||
// The control plane waits for the agent to fetch and check the
|
||||
// release, up to five minutes; fifteen seconds gave up on a node
|
||||
// that was upgrading.
|
||||
if err := client().patient(6*time.Minute).post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil {
|
||||
failed = append(failed, fmt.Sprintf("%s: %v", n.ID, err))
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
// The control plane waits for the agent's answer, or for it to
|
||||
// come back on v, longer than the agent gives its download.
|
||||
err := client().patient(agentsvc.UpgradeWait+time.Minute).post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if err != nil {
|
||||
failed = append(failed, fmt.Sprintf("%s: %v", n.ID, err))
|
||||
fmt.Fprintf(out, " %s: not upgraded: %v\n", n.ID, err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(out, " %s: on %s, coming back\n", n.ID, v)
|
||||
}()
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("not told to upgrade:\n %s\n`llmgw ctl logs NODE` says more", strings.Join(failed, "\n "))
|
||||
}
|
||||
waiting := slices.Clone(ids)
|
||||
wg.Wait()
|
||||
waiting := slices.DeleteFunc(slices.Clone(ids), func(id string) bool {
|
||||
return slices.ContainsFunc(failed, func(f string) bool { return strings.HasPrefix(f, id+": ") })
|
||||
})
|
||||
deadline := time.Now().Add(perNode)
|
||||
for {
|
||||
backAt := map[string]time.Time{}
|
||||
said := time.Now()
|
||||
for len(waiting) > 0 {
|
||||
var now []adminapi.NodeView
|
||||
_ = client().get(ctx, "/api/v1/nodes", &now)
|
||||
after := readyReplicasByNode(ctx, client)
|
||||
var reps []adminapi.ReplicaView
|
||||
_ = client().get(ctx, "/api/v1/replicas", &reps)
|
||||
after, starting := replicasByNode(reps)
|
||||
state := map[string]adminapi.NodeView{}
|
||||
for _, x := range now {
|
||||
state[x.ID] = x
|
||||
}
|
||||
waiting = slices.DeleteFunc(waiting, func(id string) bool {
|
||||
cur := state[id]
|
||||
return cur.AgentVersion == v && cur.State == "ready" && after[id] >= before[id]
|
||||
// Connected and on v: ready, or cordoned or draining as it
|
||||
// was, which an upgrade does not undo.
|
||||
if cur.AgentVersion != v || (cur.State != "ready" && cur.State != "cordoned" && cur.State != "draining") {
|
||||
return false
|
||||
}
|
||||
if _, ok := backAt[id]; !ok {
|
||||
backAt[id] = time.Now()
|
||||
}
|
||||
// Serving what it served; or, a while on, nothing on it still
|
||||
// starting: a model that went idle meanwhile is not one lost.
|
||||
return after[id] >= before[id] || (time.Since(backAt[id]) > 90*time.Second && starting[id] == 0)
|
||||
})
|
||||
if len(waiting) == 0 {
|
||||
fmt.Fprintf(out, " done in %s\n", time.Since(start).Round(time.Second))
|
||||
return nil
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
var why []string
|
||||
@@ -345,11 +386,49 @@ func upgradeWave(ctx context.Context, out io.Writer, client func() *ctlClient, v
|
||||
why = append(why, fmt.Sprintf("%s (version %q, state %q, replicas %d/%d)",
|
||||
id, cur.AgentVersion, cur.State, after[id], before[id]))
|
||||
}
|
||||
return fmt.Errorf("not back within %s:\n %s\n`llmgw ctl logs NODE` says why, and each keeps its "+
|
||||
"previous binary beside the new one", perNode, strings.Join(why, "\n "))
|
||||
failed = append(failed, fmt.Sprintf("not back within %s: %s", perNode, strings.Join(why, ", ")))
|
||||
break
|
||||
}
|
||||
if time.Since(said) >= 30*time.Second {
|
||||
var w []string
|
||||
for _, id := range waiting {
|
||||
cur := state[id]
|
||||
w = append(w, fmt.Sprintf("%s (%s, %s, replicas %d/%d)", id, orUnknown(cur.AgentVersion), cur.State,
|
||||
after[id], before[id]))
|
||||
}
|
||||
fmt.Fprintf(out, " waiting for %s\n", abbreviate(w, 8))
|
||||
said = time.Now()
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("not upgraded:\n %s\n`llmgw ctl logs NODE` says why; each keeps its previous binary beside the new one",
|
||||
strings.Join(failed, "\n "))
|
||||
}
|
||||
fmt.Fprintf(out, " done in %s\n", time.Since(start).Round(time.Second))
|
||||
return nil
|
||||
}
|
||||
|
||||
// replicasByNode counts each node's ready replicas, and those still placing
|
||||
// or starting.
|
||||
func replicasByNode(reps []adminapi.ReplicaView) (ready, starting map[string]int) {
|
||||
ready, starting = map[string]int{}, map[string]int{}
|
||||
for _, r := range reps {
|
||||
seen := map[string]bool{}
|
||||
for _, sh := range r.Shards {
|
||||
if seen[sh.Node] {
|
||||
continue
|
||||
}
|
||||
seen[sh.Node] = true
|
||||
switch r.State {
|
||||
case "ready":
|
||||
ready[sh.Node]++
|
||||
case "placing", "starting":
|
||||
starting[sh.Node]++
|
||||
}
|
||||
}
|
||||
}
|
||||
return ready, starting
|
||||
}
|
||||
|
||||
// abbreviate lists up to n names, and says how many more.
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -21,6 +22,8 @@ type fakeFleet struct {
|
||||
versions map[string]string
|
||||
stuck map[string]bool
|
||||
told []string
|
||||
// states are nodes' states other than ready.
|
||||
states map[string]string
|
||||
}
|
||||
|
||||
func (f *fakeFleet) server(t *testing.T) *httptest.Server {
|
||||
@@ -31,7 +34,11 @@ func (f *fakeFleet) server(t *testing.T) *httptest.Server {
|
||||
case r.URL.Path == "/api/v1/nodes":
|
||||
var out []adminapi.NodeView
|
||||
for id, v := range f.versions {
|
||||
out = append(out, adminapi.NodeView{ID: id, State: "ready", AgentVersion: v})
|
||||
st := "ready"
|
||||
if f.states[id] != "" {
|
||||
st = f.states[id]
|
||||
}
|
||||
out = append(out, adminapi.NodeView{ID: id, State: st, AgentVersion: v})
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(out)
|
||||
case r.URL.Path == "/api/v1/replicas":
|
||||
@@ -201,3 +208,23 @@ func TestUpgradingAControlPlaneAloneSaysItsNodesWereNot(t *testing.T) {
|
||||
t.Errorf("said something with every node on the version: %s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A cordoned node upgraded is done, as it was; a node not connected is passed
|
||||
// over rather than stopping every node after it.
|
||||
func TestAnUpgradeTakesACordonedNodeAndPassesOverOneAway(t *testing.T) {
|
||||
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.0", "c": "v1.0.0"},
|
||||
states: map[string]string{"a": "lost", "b": "cordoned"}}
|
||||
srv := f.server(t)
|
||||
var out bytes.Buffer
|
||||
err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 5*time.Second, 2)
|
||||
if slices.Contains(f.told, "a") || !slices.Contains(f.told, "b") || !slices.Contains(f.told, "c") {
|
||||
t.Errorf("told %v", f.told)
|
||||
}
|
||||
if !strings.Contains(out.String(), "Passed over, not connected: a") {
|
||||
t.Errorf("output:\n%s", out.String())
|
||||
}
|
||||
// The one away is named at the end, behind.
|
||||
if err == nil || !strings.Contains(err.Error(), "a (v1.0.0, lost)") {
|
||||
t.Errorf("err %v", err)
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,9 @@ const agentUnit = `[Unit]
|
||||
Description=llmgw node agent
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
# Never given up on: a new release that fails its first starts, or an upgrade
|
||||
# that restarts it twice in a minute, would otherwise leave the node off.
|
||||
StartLimitIntervalSec=0
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/local/bin/llmgw agent --config /etc/llmgw/agent.yaml
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
@@ -571,9 +572,15 @@ func (m consoleModel) nodeMenu(n adminapi.NodeView) *menuDialog {
|
||||
menuItem{"Upgrade its agent…", func(m *consoleModel) tea.Cmd {
|
||||
m.openForm(&formDialog{title: "Upgrade " + n.ID,
|
||||
note: "It fetches the release for its platform, checks it, swaps it in and restarts; engines keep serving.",
|
||||
fields: []formField{textF("version", version.Version, "a release, like v0.11.1")},
|
||||
fields: []formField{textF("version", releaseOrEmpty(version.Version), "a release, like v0.11.1")},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
return m.call("Upgrade "+n.ID, http.MethodPost, path+"/upgrade", adminapi.UpgradeRequest{Version: v[0]}, nil), nil
|
||||
if !release.Tag.MatchString(v[0]) {
|
||||
return nil, fmt.Errorf("a release is vMAJOR.MINOR.PATCH, like v0.15.3")
|
||||
}
|
||||
// As long as the control plane waits: the download, the
|
||||
// restart, the node back on the version.
|
||||
return m.callWaiting("Upgrade "+n.ID, http.MethodPost, path+"/upgrade", adminapi.UpgradeRequest{Version: v[0]},
|
||||
func(json.RawMessage) string { return n.ID + " is on " + v[0] + "." }, agentsvc.UpgradeWait+time.Minute), nil
|
||||
}})
|
||||
return nil
|
||||
}},
|
||||
@@ -1056,3 +1063,12 @@ func purgeSays(raw json.RawMessage) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// releaseOrEmpty is this build's version when it is a release, else empty:
|
||||
// a development build's is not one a node can fetch.
|
||||
func releaseOrEmpty(v string) string {
|
||||
if release.Tag.MatchString(v) {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Reference in new issue
Block a user