Upgrade the cluster with one command, over the agents' own streams
Every upgrade so far has been a script of mine copying a binary to each machine over ssh, restarting it and polling until it came back — which is the job, but not something the product could do, and which needed ssh into every node. `sudo llmgw ctl upgrade VERSION` on the control plane does it. This machine first: the release for its platform, checked against the release's checksums, the old binary kept beside it, llmgw-server restarted, and a wait until every node that was ready has reconnected. Then each node in turn: the control plane tells its agent the version over the stream it already holds; the agent fetches and checks the release itself, swaps it in with a rename, answers, and exits with status 75 so systemd starts the new binary. Engines keep running, as for any agent restart, and are adopted again. A node is done when it is back on the new version, ready, with as many ready replicas as before; one that is not within the timeout stops the run before the next node, so a bad release reaches one machine. Nodes already on the version are skipped, so running it again resumes. The fetch-and-verify is one package now, shared with onboarding's idea of what a release version looks like. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
3bf4b7b709
commit
116e75f8cb
19 files changed
+1082
-90
No files matched your search
@@ -8,6 +8,7 @@ import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
@@ -18,6 +19,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -1119,6 +1121,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
|
||||
a.removeWeights(m.RemoveWeights.GetPaths())
|
||||
case *llmgwv1.ControlMessage_Provision:
|
||||
a.setProvision(m.Provision)
|
||||
case *llmgwv1.ControlMessage_Upgrade:
|
||||
go a.upgrade(ctx, m.Upgrade, out)
|
||||
case *llmgwv1.ControlMessage_Logs:
|
||||
reply := a.logsReply(m.Logs)
|
||||
// Off the receive loop: a full send queue must not stop commands being
|
||||
@@ -1874,3 +1878,49 @@ func (a *Agent) sampleNVLink(hb *llmgwv1.Heartbeat) {
|
||||
m := discoverIMEX(ctx)
|
||||
hb.Imex = &llmgwv1.IMEX{State: m.State, Channels: int32(m.Channels)}
|
||||
}
|
||||
|
||||
// upgradeExit is the exit status after an upgrade: non-zero, so systemd's
|
||||
// Restart=on-failure starts the new binary, and distinct, so the journal says
|
||||
// why the agent went.
|
||||
const upgradeExit = 75
|
||||
|
||||
// exitForUpgrade ends the process; a variable so a test can see it called
|
||||
// rather than have its own process end.
|
||||
var exitForUpgrade = func() { os.Exit(upgradeExit) }
|
||||
|
||||
// upgradeTarget is the binary an upgrade replaces; a variable so a test
|
||||
// replaces a file of its own rather than the test binary running it.
|
||||
var upgradeTarget = os.Executable
|
||||
|
||||
// upgrade replaces this agent's binary with a release and restarts into it.
|
||||
//
|
||||
// Engines are not touched: an agent restart leaves them running and adopts
|
||||
// them again, which is what makes upgrading a node something that can be done
|
||||
// while it serves.
|
||||
func (a *Agent) upgrade(ctx context.Context, m *llmgwv1.UpgradeAgent, out chan<- *llmgwv1.AgentMessage) {
|
||||
reply := &llmgwv1.UpgradeReply{RequestId: m.GetRequestId(), Version: m.GetVersion()}
|
||||
send := func() {
|
||||
select {
|
||||
case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: reply}}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}
|
||||
a.log.Info("upgrading", "to", m.GetVersion(), "from", a.cfg.Version)
|
||||
self, err := upgradeTarget()
|
||||
if err == nil {
|
||||
dctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
_, err = release.Install(dctx, nil, m.GetReleases(), m.GetVersion(), runtime.GOOS, runtime.GOARCH, self)
|
||||
cancel()
|
||||
}
|
||||
if err != nil {
|
||||
reply.Error = err.Error()
|
||||
a.log.Error("upgrade failed; still running the old version", "to", m.GetVersion(), "err", err)
|
||||
send()
|
||||
return
|
||||
}
|
||||
send()
|
||||
a.log.Info("upgraded; restarting into the new version", "to", m.GetVersion())
|
||||
// A moment for the reply to leave before the stream goes.
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
exitForUpgrade()
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
)
|
||||
|
||||
func TestAnAgentUpgradesItselfAndSaysSoBeforeRestarting(t *testing.T) {
|
||||
body := []byte("the new agent")
|
||||
sum := sha256.Sum256(body)
|
||||
asset := release.AssetName("v9.9.9", runtime.GOOS, runtime.GOARCH)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/v9.9.9/checksums.txt":
|
||||
_, _ = w.Write([]byte(hex.EncodeToString(sum[:]) + " " + asset + "\n"))
|
||||
case "/v9.9.9/" + asset:
|
||||
_, _ = w.Write(body)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
bin := filepath.Join(t.TempDir(), "llmgw")
|
||||
_ = os.WriteFile(bin, []byte("the old agent"), 0o755)
|
||||
exited := make(chan struct{}, 1)
|
||||
oldTarget, oldExit := upgradeTarget, exitForUpgrade
|
||||
upgradeTarget = func() (string, error) { return bin, nil }
|
||||
exitForUpgrade = func() { exited <- struct{}{} }
|
||||
t.Cleanup(func() { upgradeTarget, exitForUpgrade = oldTarget, oldExit })
|
||||
|
||||
cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(), EnginesDir: t.TempDir()}
|
||||
cfg.Superv.WorkDir = t.TempDir()
|
||||
a, err := New(cfg, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := make(chan *llmgwv1.AgentMessage, 1)
|
||||
go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u1", Version: "v9.9.9", Releases: srv.URL}, out)
|
||||
|
||||
select {
|
||||
case m := <-out:
|
||||
if r := m.GetUpgrade(); r.GetRequestId() != "u1" || r.GetError() != "" {
|
||||
t.Fatalf("reply %+v", r)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("no reply")
|
||||
}
|
||||
select {
|
||||
case <-exited:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the agent did not restart into the new binary")
|
||||
}
|
||||
if b, _ := os.ReadFile(bin); string(b) != "the new agent" {
|
||||
t.Errorf("binary holds %q", b)
|
||||
}
|
||||
|
||||
// A version with no release answers with why, and does not restart.
|
||||
go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u2", Version: "v9.9.8", Releases: srv.URL}, out)
|
||||
if r := (<-out).GetUpgrade(); r.GetError() == "" {
|
||||
t.Error("an upgrade to a release that does not exist reported success")
|
||||
}
|
||||
select {
|
||||
case <-exited:
|
||||
t.Error("a failed upgrade restarted the agent")
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
@@ -141,6 +141,7 @@ func newCtlCmd() *cobra.Command {
|
||||
newScaleCmd(client),
|
||||
newRestartCmd(client),
|
||||
newRetryCmd(client),
|
||||
newUpgradeCmd(client),
|
||||
newEngineCheckCmd(),
|
||||
newToolCheckCmd(),
|
||||
newWeightsCmd(client, &asJSON),
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
)
|
||||
|
||||
func newUpgradeCmd(client func() *ctlClient) *cobra.Command {
|
||||
var (
|
||||
nodes []string
|
||||
skipCP bool
|
||||
releases string
|
||||
perNode time.Duration
|
||||
serverSvc string
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "upgrade VERSION",
|
||||
Short: "Upgrade this control plane, then every node, one at a time",
|
||||
Long: "Moves the cluster to a release: v0.10.4, or a candidate like v0.10.4-rc.2.\n\n" +
|
||||
"This machine first — run it with sudo on the control plane, since it\n" +
|
||||
"replaces /usr/local/bin/llmgw and restarts llmgw-server. Then each node, in\n" +
|
||||
"turn: its agent fetches the release for its own platform, checks it\n" +
|
||||
"against the release's checksums, swaps it in and restarts. Engines keep\n" +
|
||||
"serving through all of it and are adopted again; no ssh into the nodes is\n" +
|
||||
"needed.\n\n" +
|
||||
"A node counts as done when it is back on the new version, ready, with as\n" +
|
||||
"many ready replicas as before. One that is not within --timeout stops the\n" +
|
||||
"upgrade there, so a bad release reaches one node rather than all of them.\n" +
|
||||
"Nodes already on the version are skipped, so running it again resumes.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
v := args[0]
|
||||
if !release.Tag.MatchString(v) {
|
||||
return fmt.Errorf("%q is not a release version (v1.2.3 or v1.2.3-rc.4)", v)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
ctx := cmd.Context()
|
||||
if !skipCP {
|
||||
if err := upgradeControlPlane(ctx, out, client, v, releases, serverSvc); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return upgradeNodes(ctx, out, client, v, nodes, perNode)
|
||||
},
|
||||
}
|
||||
c.Flags().StringSliceVar(&nodes, "nodes", nil, "only these nodes (default: all)")
|
||||
c.Flags().BoolVar(&skipCP, "skip-control-plane", false, "leave this machine as it is and upgrade only nodes")
|
||||
c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from, for this machine")
|
||||
c.Flags().DurationVar(&perNode, "timeout", 3*time.Minute, "how long a node may take to come back")
|
||||
c.Flags().StringVar(&serverSvc, "service", "llmgw-server", "this machine's systemd unit")
|
||||
return c
|
||||
}
|
||||
|
||||
func upgradeControlPlane(ctx context.Context, out io.Writer, client func() *ctlClient, v, releases, svc string) error {
|
||||
if version.Version == v {
|
||||
fmt.Fprintf(out, "this machine already runs %s\n", v)
|
||||
return nil
|
||||
}
|
||||
if os.Geteuid() != 0 {
|
||||
return errors.New("replacing this machine's binary needs root: run it with sudo, " +
|
||||
"or pass --skip-control-plane to upgrade only the nodes")
|
||||
}
|
||||
self, err := os.Executable()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, "== this machine: %s -> %s\n", version.Version, v)
|
||||
backup, err := release.Install(ctx, nil, releases, v, runtime.GOOS, runtime.GOARCH, self)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, " installed, checksum verified; the old binary is %s\n", backup)
|
||||
var before []adminapi.NodeView
|
||||
_ = client().get(ctx, "/api/v1/nodes", &before)
|
||||
if b, err := exec.CommandContext(ctx, "systemctl", "restart", svc).CombinedOutput(); err != nil { //nolint:gosec // fixed command
|
||||
return fmt.Errorf("restarting %s: %v: %s", svc, err, strings.TrimSpace(string(b)))
|
||||
}
|
||||
fmt.Fprintf(out, " %s restarted; waiting for its nodes to reconnect\n", svc)
|
||||
// Back when the admin API answers and every node that was ready is ready
|
||||
// again: agents reconnect within seconds of the control plane returning.
|
||||
want := 0
|
||||
for _, n := range before {
|
||||
if n.State == "ready" {
|
||||
want++
|
||||
}
|
||||
}
|
||||
deadline := time.Now().Add(2 * time.Minute)
|
||||
for {
|
||||
var now []adminapi.NodeView
|
||||
if err := client().get(ctx, "/api/v1/nodes", &now); err == nil {
|
||||
ready := 0
|
||||
for _, n := range now {
|
||||
if n.State == "ready" {
|
||||
ready++
|
||||
}
|
||||
}
|
||||
if ready >= want {
|
||||
fmt.Fprintf(out, " back: %d of %d nodes ready\n", ready, len(now))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("%s did not come back with its %d nodes within two minutes; "+
|
||||
"see journalctl -u %s, and %s holds the previous binary", svc, want, svc, backup)
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient, v string, only []string, perNode time.Duration) error {
|
||||
var nodes []adminapi.NodeView
|
||||
if err := client().get(ctx, "/api/v1/nodes", &nodes); err != nil {
|
||||
return err
|
||||
}
|
||||
slices.SortFunc(nodes, func(a, b adminapi.NodeView) int { return strings.Compare(a.ID, b.ID) })
|
||||
done, skipped := 0, 0
|
||||
for _, n := range nodes {
|
||||
if len(only) > 0 && !slices.Contains(only, n.ID) {
|
||||
continue
|
||||
}
|
||||
if n.AgentVersion == v {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
start := time.Now()
|
||||
before := readyReplicasOn(ctx, client, n.ID)
|
||||
fmt.Fprintf(out, "== %s: %s -> %s\n", n.ID, n.AgentVersion, v)
|
||||
body, _ := json.Marshal(adminapi.UpgradeRequest{Version: v})
|
||||
if err := client().post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil {
|
||||
return fmt.Errorf("%s: %w\nstopped here; `llmgw ctl logs %s` says more", n.ID, err, n.ID)
|
||||
}
|
||||
fmt.Fprintf(out, " new binary in place; waiting for it to come back with its %d replica(s)\n", before)
|
||||
deadline := time.Now().Add(perNode)
|
||||
for {
|
||||
var now []adminapi.NodeView
|
||||
_ = client().get(ctx, "/api/v1/nodes", &now)
|
||||
var cur adminapi.NodeView
|
||||
for _, x := range now {
|
||||
if x.ID == n.ID {
|
||||
cur = x
|
||||
}
|
||||
}
|
||||
after := readyReplicasOn(ctx, client, n.ID)
|
||||
if cur.AgentVersion == v && cur.State == "ready" && after >= before {
|
||||
fmt.Fprintf(out, " done in %s: ready, replicas %d/%d\n",
|
||||
time.Since(start).Round(time.Second), after, before)
|
||||
done++
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("%s did not come back within %s (version %q, state %q, replicas %d/%d)\n"+
|
||||
"stopped here, before the next node; `llmgw ctl logs %s` says why, and the node "+
|
||||
"keeps its previous binary beside the new one", n.ID, perNode, cur.AgentVersion,
|
||||
cur.State, after, before, n.ID)
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(out, "%d node(s) upgraded to %s, %d already on it\n", done, v, skipped)
|
||||
return nil
|
||||
}
|
||||
|
||||
// readyReplicasOn counts the ready replicas with a shard on a node.
|
||||
func readyReplicasOn(ctx context.Context, client func() *ctlClient, node string) int {
|
||||
var reps []adminapi.ReplicaView
|
||||
if err := client().get(ctx, "/api/v1/replicas", &reps); err != nil {
|
||||
return 0
|
||||
}
|
||||
n := 0
|
||||
for _, r := range reps {
|
||||
if r.State != "ready" {
|
||||
continue
|
||||
}
|
||||
for _, s := range r.Shards {
|
||||
if s.Node == node {
|
||||
n++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// fakeFleet is an admin API whose nodes come back on the version they are told
|
||||
// to upgrade to — except those listed in stuck.
|
||||
type fakeFleet struct {
|
||||
mu sync.Mutex
|
||||
versions map[string]string
|
||||
stuck map[string]bool
|
||||
told []string
|
||||
}
|
||||
|
||||
func (f *fakeFleet) server(t *testing.T) *httptest.Server {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
switch {
|
||||
case r.URL.Path == "/api/v1/nodes":
|
||||
var out []adminapi.NodeView
|
||||
for id, v := range f.versions {
|
||||
out = append(out, adminapi.NodeView{ID: id, State: "ready", AgentVersion: v})
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(out)
|
||||
case r.URL.Path == "/api/v1/replicas":
|
||||
var out []adminapi.ReplicaView
|
||||
for id := range f.versions {
|
||||
out = append(out, adminapi.ReplicaView{ID: "r-" + id, State: "ready",
|
||||
Shards: []adminapi.ShardView{{Node: id}}})
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(out)
|
||||
case strings.HasSuffix(r.URL.Path, "/upgrade"):
|
||||
id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/nodes/"), "/upgrade")
|
||||
var req adminapi.UpgradeRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
f.told = append(f.told, id)
|
||||
if !f.stuck[id] {
|
||||
f.versions[id] = req.Version
|
||||
}
|
||||
_, _ = w.Write([]byte(`{}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestAnUpgradeGoesNodeByNodeAndSkipsTheDone(t *testing.T) {
|
||||
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.1", "c": "v1.0.0"}}
|
||||
srv := f.server(t)
|
||||
var out bytes.Buffer
|
||||
err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(f.told, ",") != "a,c" {
|
||||
t.Errorf("told %v; b was already on the version", f.told)
|
||||
}
|
||||
if !strings.Contains(out.String(), "2 node(s) upgraded to v1.0.1, 1 already on it") {
|
||||
t.Errorf("output:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpgradeStopsAtTheFirstNodeThatDoesNotComeBack(t *testing.T) {
|
||||
// A bad release reaches one node, not all of them.
|
||||
f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.0"}, stuck: map[string]bool{"a": true}}
|
||||
srv := f.server(t)
|
||||
var out bytes.Buffer
|
||||
err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 3*time.Second)
|
||||
if err == nil || !strings.Contains(err.Error(), "a did not come back") {
|
||||
t.Fatalf("err %v", err)
|
||||
}
|
||||
if len(f.told) != 1 {
|
||||
t.Errorf("went on to %v after a node failed to come back", f.told)
|
||||
}
|
||||
}
|
||||
@@ -173,6 +173,7 @@ func newServerCmd() *cobra.Command {
|
||||
}
|
||||
if admin != nil {
|
||||
api := adminapi.New(admin)
|
||||
api.SetReleases(cfg.Releases)
|
||||
// Catalog changes need somewhere to persist, so this is wired
|
||||
// only when there is a store. Without one, the endpoints say so
|
||||
// rather than accepting a model that vanishes on restart.
|
||||
@@ -546,6 +547,19 @@ func (r *dispatcherRef) FabricTest(ctx context.Context, nodeID string, req *llmg
|
||||
return ft.FabricTest(ctx, nodeID, req)
|
||||
}
|
||||
|
||||
// Upgrade forwards a node upgrade, as Logs does.
|
||||
func (r *dispatcherRef) Upgrade(ctx context.Context, nodeID, version, releases string) error {
|
||||
d, err := r.get()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
u, ok := d.(control.Upgrader)
|
||||
if !ok {
|
||||
return errors.New("this dispatcher cannot upgrade nodes")
|
||||
}
|
||||
return u.Upgrade(ctx, nodeID, version, releases)
|
||||
}
|
||||
|
||||
func (r *dispatcherRef) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
|
||||
d, err := r.get()
|
||||
if err != nil {
|
||||
|
||||
@@ -86,6 +86,11 @@ type Server struct {
|
||||
// Empty means /var/backups/llmgw-server.
|
||||
BackupDir string `koanf:"backup_dir"`
|
||||
|
||||
// Releases is where `llmgw ctl upgrade` fetches release binaries from,
|
||||
// for this machine and — through their agents — every node. Empty means
|
||||
// the project's own releases.
|
||||
Releases string `koanf:"releases"`
|
||||
|
||||
Router Router `koanf:"router"`
|
||||
Control Control `koanf:"control"`
|
||||
Models []catalog.Model `koanf:"models"`
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
@@ -30,6 +31,9 @@ import (
|
||||
|
||||
// API serves the management endpoints.
|
||||
type API struct {
|
||||
// releases is where release binaries are fetched from, for upgrades.
|
||||
releases string
|
||||
|
||||
models Models
|
||||
capacity Capacity
|
||||
usage Usage
|
||||
@@ -74,6 +78,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/weights/collect", a.collectWeights)
|
||||
mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/upgrade", a.upgradeNode)
|
||||
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
|
||||
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
||||
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
||||
@@ -1165,3 +1170,37 @@ func (a *API) request(w http.ResponseWriter, r *http.Request) {
|
||||
PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros,
|
||||
})
|
||||
}
|
||||
|
||||
// SetReleases sets where upgrades fetch release binaries from.
|
||||
func (a *API) SetReleases(base string) { a.releases = base }
|
||||
|
||||
// UpgradeRequest names the release a node should run.
|
||||
type UpgradeRequest struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
func (a *API) upgradeNode(w http.ResponseWriter, r *http.Request) {
|
||||
var req UpgradeRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the upgrade request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
if !release.Tag.MatchString(req.Version) {
|
||||
writeError(w, http.StatusBadRequest, req.Version+" is not a release version")
|
||||
return
|
||||
}
|
||||
u := a.ctl.Upgrader()
|
||||
if u == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane cannot upgrade nodes")
|
||||
return
|
||||
}
|
||||
base := a.releases
|
||||
if base == "" {
|
||||
base = release.DefaultBase
|
||||
}
|
||||
if err := u.Upgrade(r.Context(), r.PathValue("id"), req.Version, base); err != nil {
|
||||
writeError(w, http.StatusBadGateway, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"node": r.PathValue("id"), "version": req.Version})
|
||||
}
|
||||
@@ -264,3 +264,22 @@ func TestAGPUFallingOutOfItsDomainTakesTheNodeOut(t *testing.T) {
|
||||
t.Errorf("imex = %+v", n.IMEX)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpgradeWaitsForTheAgentsAnswer(t *testing.T) {
|
||||
s := newTestService()
|
||||
c := &conn{nodeID: "n1", send: make(chan *llmgwv1.ControlMessage, 4), closed: make(chan struct{})}
|
||||
s.conns["n1"] = c
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- s.Upgrade(t.Context(), "n1", "v1.2.3", "https://r") }()
|
||||
|
||||
req := (<-c.send).GetUpgrade()
|
||||
if req.GetVersion() != "v1.2.3" || req.GetReleases() != "https://r" {
|
||||
t.Fatalf("sent %v", req)
|
||||
}
|
||||
s.handle("n1", &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: &llmgwv1.UpgradeReply{
|
||||
RequestId: req.GetRequestId(), Version: "v1.2.3", Error: "checksum mismatch",
|
||||
}}})
|
||||
if err := <-done; err == nil || err.Error() != "checksum mismatch" {
|
||||
t.Errorf("the agent's refusal was not passed on: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -60,6 +60,8 @@ type Service struct {
|
||||
logMu sync.Mutex
|
||||
logWaits map[string]chan *llmgwv1.LogsReply
|
||||
logSeq uint64
|
||||
// upgradeWaits are the upgrade requests awaiting a reply; guarded by logMu.
|
||||
upgradeWaits map[string]chan *llmgwv1.UpgradeReply
|
||||
// scanWaits are the fabric scans awaiting a reply, by request id.
|
||||
scanWaits map[string]chan *llmgwv1.FabricScanReply
|
||||
testWaits map[string]chan *llmgwv1.FabricTestReply
|
||||
@@ -309,6 +311,15 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
|
||||
ch <- m.FabricScan
|
||||
}
|
||||
|
||||
case *llmgwv1.AgentMessage_Upgrade:
|
||||
s.logMu.Lock()
|
||||
ch, ok := s.upgradeWaits[m.Upgrade.GetRequestId()]
|
||||
delete(s.upgradeWaits, m.Upgrade.GetRequestId())
|
||||
s.logMu.Unlock()
|
||||
if ok {
|
||||
ch <- m.Upgrade
|
||||
}
|
||||
|
||||
case *llmgwv1.AgentMessage_Logs:
|
||||
s.logMu.Lock()
|
||||
ch, ok := s.logWaits[m.Logs.GetRequestId()]
|
||||
@@ -842,3 +853,44 @@ func installable(reg *llmgwv1.Register) []string {
|
||||
// The admin API finds the bandwidth test by asserting this; a signature that
|
||||
// drifted would make it quietly nil.
|
||||
var _ fabrictest.Asker = (*Service)(nil)
|
||||
|
||||
// Upgrade tells a node's agent to replace its binary with a release and
|
||||
// restart, and returns once the agent says the new binary is in place — or
|
||||
// why it is not. The agent then exits and comes back on the new version; the
|
||||
// caller watches for that in the inventory.
|
||||
func (s *Service) Upgrade(ctx context.Context, nodeID, version, releases string) error {
|
||||
s.logMu.Lock()
|
||||
if s.upgradeWaits == nil {
|
||||
s.upgradeWaits = map[string]chan *llmgwv1.UpgradeReply{}
|
||||
}
|
||||
s.logSeq++
|
||||
id := fmt.Sprintf("upgrade-%d", s.logSeq)
|
||||
ch := make(chan *llmgwv1.UpgradeReply, 1)
|
||||
s.upgradeWaits[id] = ch
|
||||
s.logMu.Unlock()
|
||||
defer func() {
|
||||
s.logMu.Lock()
|
||||
delete(s.upgradeWaits, id)
|
||||
s.logMu.Unlock()
|
||||
}()
|
||||
|
||||
if err := s.send(nodeID, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_Upgrade{
|
||||
Upgrade: &llmgwv1.UpgradeAgent{RequestId: id, Version: version, Releases: releases},
|
||||
}}); err != nil {
|
||||
return err
|
||||
}
|
||||
s.log.Info("told a node to upgrade", "node", nodeID, "version", version)
|
||||
// A download of some tens of megabytes on a slow link, and no longer: an
|
||||
// agent too old to know this message never answers.
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
select {
|
||||
case r := <-ch:
|
||||
if r.GetError() != "" {
|
||||
return errors.New(r.GetError())
|
||||
}
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("node %s did not answer the upgrade; an agent older than v0.10.4 cannot upgrade itself", nodeID)
|
||||
}
|
||||
}
|
||||
@@ -211,6 +211,17 @@ func (c *Controller) WeightRemover() WeightRemover {
|
||||
return r
|
||||
}
|
||||
|
||||
// Upgrader can tell a node's agent to upgrade itself.
|
||||
type Upgrader interface {
|
||||
Upgrade(ctx context.Context, nodeID, version, releases string) error
|
||||
}
|
||||
|
||||
// Upgrader returns the dispatcher's upgrade capability, or nil.
|
||||
func (c *Controller) Upgrader() Upgrader {
|
||||
u, _ := c.disp.(Upgrader)
|
||||
return u
|
||||
}
|
||||
|
||||
// LogReader can ask a node for the recent lines of one of its logs.
|
||||
type LogReader interface {
|
||||
Logs(ctx context.Context, nodeID, source string, lines int) (out, sources []string, err error)
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -335,7 +336,7 @@ func certNames(path string) ([]string, error) {
|
||||
// releaseTag is a version CI published: v0.10.4, or a release candidate of
|
||||
// it, v0.10.4-rc.2. Anything else — a build with commits past its tag — has no
|
||||
// release to download.
|
||||
var releaseTag = regexp.MustCompile(`^v\d+\.\d+\.\d+(-rc\.\d+)?$`)
|
||||
var releaseTag = release.Tag
|
||||
|
||||
func (r *Runner) binary(ctx context.Context, res *Result) error {
|
||||
v := r.Opts.Version
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
// Package release fetches and installs llmgw's own release binaries.
|
||||
//
|
||||
// One implementation for everywhere a binary is replaced — the control plane
|
||||
// upgrading itself, and an agent told to upgrade by it — so the checks are
|
||||
// the same on both: the release's own checksum is verified before anything is
|
||||
// installed, the old binary is kept beside the new one, and the swap is a
|
||||
// rename, so a machine is never left without a binary that runs.
|
||||
package release
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultBase is where CI publishes releases.
|
||||
const DefaultBase = "https://gitea.mynodes.xyz/ilker/llmgw/releases/download"
|
||||
|
||||
// Tag is a version CI published: v0.10.4, or a release candidate v0.10.4-rc.2.
|
||||
var Tag = regexp.MustCompile(`^v\d+\.\d+\.\d+(-rc\.\d+)?$`)
|
||||
|
||||
// AssetName is the release file for a platform.
|
||||
func AssetName(version, goos, goarch string) string {
|
||||
return "llmgw-" + version + "-" + goos + "-" + goarch
|
||||
}
|
||||
|
||||
// Install fetches version for this platform from base, checks it against the
|
||||
// release's checksums, and puts it at target, keeping the old one as
|
||||
// target.bak-pre-<version>. It returns the backup's path.
|
||||
func Install(ctx context.Context, client *http.Client, base, version, goos, goarch, target string) (string, error) {
|
||||
if !Tag.MatchString(version) {
|
||||
return "", fmt.Errorf("%q is not a release version", version)
|
||||
}
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 10 * time.Minute}
|
||||
}
|
||||
target, err := filepath.EvalSymlinks(target)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
dir := filepath.Dir(target)
|
||||
asset := AssetName(version, goos, goarch)
|
||||
root := strings.TrimSuffix(base, "/") + "/" + version + "/"
|
||||
|
||||
want, err := checksum(ctx, client, root+"checksums.txt", asset)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Beside the target, so the final rename is on one filesystem and atomic.
|
||||
tmp, err := os.CreateTemp(dir, ".llmgw-"+version+"-*")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
got, err := download(ctx, client, root+asset, tmp)
|
||||
if cerr := tmp.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if got != want {
|
||||
return "", fmt.Errorf("%s does not match the release's checksum (got %s, expected %s)",
|
||||
asset, got[:12], want[:12])
|
||||
}
|
||||
if err := os.Chmod(tmp.Name(), 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
backup := target + ".bak-pre-" + version
|
||||
if err := copyFile(target, backup); err != nil {
|
||||
return "", fmt.Errorf("keeping the old binary: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp.Name(), target); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return backup, nil
|
||||
}
|
||||
|
||||
func checksum(ctx context.Context, client *http.Client, url, asset string) (string, error) {
|
||||
body, err := get(ctx, client, url)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer body.Close()
|
||||
sc := bufio.NewScanner(io.LimitReader(body, 1<<20))
|
||||
for sc.Scan() {
|
||||
f := strings.Fields(sc.Text())
|
||||
if len(f) == 2 && f[1] == asset && len(f[0]) == 64 {
|
||||
return f[0], nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("the release's checksums list no %s", asset)
|
||||
}
|
||||
|
||||
func download(ctx context.Context, client *http.Client, url string, dst io.Writer) (string, error) {
|
||||
body, err := get(ctx, client, url)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer body.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(io.MultiWriter(dst, h), body); err != nil {
|
||||
return "", fmt.Errorf("downloading %s: %w", url, err)
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func get(ctx context.Context, client *http.Client, url string) (io.ReadCloser, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetching %s: %w", url, err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("fetching %s: %s", url, resp.Status)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) error {
|
||||
in, err := os.Open(src) //nolint:gosec // our own binary
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755) //nolint:gosec // beside it
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, in); err != nil {
|
||||
out.Close()
|
||||
return err
|
||||
}
|
||||
return out.Close()
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package release
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func releaseServer(t *testing.T, version, asset string, body []byte, sum string) *httptest.Server {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/" + version + "/checksums.txt":
|
||||
_, _ = w.Write([]byte(sum + " " + asset + "\n"))
|
||||
case "/" + version + "/" + asset:
|
||||
_, _ = w.Write(body)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func TestABinaryIsReplacedOnlyWithWhatTheReleasePublished(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
target := filepath.Join(dir, "llmgw")
|
||||
_ = os.WriteFile(target, []byte("old"), 0o755)
|
||||
body := []byte("new binary")
|
||||
sum := sha256.Sum256(body)
|
||||
asset := AssetName("v1.2.3", "linux", "amd64")
|
||||
|
||||
srv := releaseServer(t, "v1.2.3", asset, body, hex.EncodeToString(sum[:]))
|
||||
backup, err := Install(t.Context(), srv.Client(), srv.URL, "v1.2.3", "linux", "amd64", target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b, _ := os.ReadFile(target); string(b) != "new binary" {
|
||||
t.Errorf("target holds %q", b)
|
||||
}
|
||||
if b, _ := os.ReadFile(backup); string(b) != "old" {
|
||||
t.Errorf("the old binary was not kept: %q", b)
|
||||
}
|
||||
|
||||
// A download that does not match its checksum changes nothing.
|
||||
bad := releaseServer(t, "v1.2.4", AssetName("v1.2.4", "linux", "amd64"), []byte("tampered"),
|
||||
hex.EncodeToString(sum[:]))
|
||||
if _, err := Install(t.Context(), bad.Client(), bad.URL, "v1.2.4", "linux", "amd64", target); err == nil {
|
||||
t.Fatal("a binary not matching its checksum was installed")
|
||||
}
|
||||
if b, _ := os.ReadFile(target); string(b) != "new binary" {
|
||||
t.Errorf("a failed install touched the binary: %q", b)
|
||||
}
|
||||
if _, err := Install(t.Context(), nil, srv.URL, "../../x", "linux", "amd64", target); err == nil {
|
||||
t.Error("a version that is not one was fetched")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user