Upgrade the cluster with one command, over the agents' own streams
Every upgrade so far has been a script of mine copying a binary to each machine over ssh, restarting it and polling until it came back — which is the job, but not something the product could do, and which needed ssh into every node. `sudo llmgw ctl upgrade VERSION` on the control plane does it. This machine first: the release for its platform, checked against the release's checksums, the old binary kept beside it, llmgw-server restarted, and a wait until every node that was ready has reconnected. Then each node in turn: the control plane tells its agent the version over the stream it already holds; the agent fetches and checks the release itself, swaps it in with a rename, answers, and exits with status 75 so systemd starts the new binary. Engines keep running, as for any agent restart, and are adopted again. A node is done when it is back on the new version, ready, with as many ready replicas as before; one that is not within the timeout stops the run before the next node, so a bad release reaches one machine. Nodes already on the version are skipped, so running it again resumes. The fetch-and-verify is one package now, shared with onboarding's idea of what a release version looks like. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
3bf4b7b709
commit
116e75f8cb
19 files changed
+1082
-90
No files matched your search
@@ -10,6 +10,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
@@ -30,6 +31,9 @@ import (
|
||||
|
||||
// API serves the management endpoints.
|
||||
type API struct {
|
||||
// releases is where release binaries are fetched from, for upgrades.
|
||||
releases string
|
||||
|
||||
models Models
|
||||
capacity Capacity
|
||||
usage Usage
|
||||
@@ -74,6 +78,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/weights/collect", a.collectWeights)
|
||||
mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/upgrade", a.upgradeNode)
|
||||
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
|
||||
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
||||
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
||||
@@ -1165,3 +1170,37 @@ func (a *API) request(w http.ResponseWriter, r *http.Request) {
|
||||
PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros,
|
||||
})
|
||||
}
|
||||
|
||||
// SetReleases sets where upgrades fetch release binaries from.
|
||||
func (a *API) SetReleases(base string) { a.releases = base }
|
||||
|
||||
// UpgradeRequest names the release a node should run.
|
||||
type UpgradeRequest struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
func (a *API) upgradeNode(w http.ResponseWriter, r *http.Request) {
|
||||
var req UpgradeRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the upgrade request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
if !release.Tag.MatchString(req.Version) {
|
||||
writeError(w, http.StatusBadRequest, req.Version+" is not a release version")
|
||||
return
|
||||
}
|
||||
u := a.ctl.Upgrader()
|
||||
if u == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane cannot upgrade nodes")
|
||||
return
|
||||
}
|
||||
base := a.releases
|
||||
if base == "" {
|
||||
base = release.DefaultBase
|
||||
}
|
||||
if err := u.Upgrade(r.Context(), r.PathValue("id"), req.Version, base); err != nil {
|
||||
writeError(w, http.StatusBadGateway, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"node": r.PathValue("id"), "version": req.Version})
|
||||
}
|
||||
Reference in new issue
Block a user