Track the registry's new weights for served models, and move a model to them or back
A model served from the registry records its origin: repository, commit and each file's checksum; one served before is worked out from the mirror's record of where it fetched the file. Every mirror.update_check (6h) the control plane asks whether those files changed there. ctl model updates lists it, ctl model update fetches beside the old files, checks, swaps and replaces replicas one at a time, ctl model rollback goes back. The console shows it on Models and offers the same. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a96b1a54b2
commit
44425193bd
15 files changed
+1393
-33
No files matched your search
@@ -121,6 +121,10 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/restart", a.restart)
|
||||
mux.HandleFunc("POST /api/v1/models/serve", a.serveModel)
|
||||
mux.HandleFunc("GET /api/v1/jobs/{id}", a.job)
|
||||
mux.HandleFunc("GET /api/v1/updates", a.listUpdates)
|
||||
mux.HandleFunc("POST /api/v1/updates/check", a.checkUpdates)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel)
|
||||
mux.HandleFunc("GET /api/v1/requests/{id}", a.request)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
|
||||
mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -336,6 +340,19 @@ type ModelView struct {
|
||||
Adapters []catalog.Adapter `json:"adapters,omitempty"`
|
||||
Preempted int `json:"preempted,omitempty"`
|
||||
HeldFor string `json:"held_for,omitempty"`
|
||||
// Origin is the registry repository its weights came from, and at which
|
||||
// commit; Update what that repository had when it was last asked.
|
||||
Origin *OriginView `json:"origin,omitempty"`
|
||||
Update *modelserve.UpdateInfo `json:"update,omitempty"`
|
||||
}
|
||||
|
||||
// OriginView is where a model's weights were published.
|
||||
type OriginView struct {
|
||||
Repo string `json:"repo"`
|
||||
Revision string `json:"revision"`
|
||||
// Previous is the commit the last update replaced, which it can go back
|
||||
// to.
|
||||
Previous string `json:"previous,omitempty"`
|
||||
}
|
||||
|
||||
// ReplicaView is a replica as `ctl replicas` shows it.
|
||||
@@ -586,6 +603,10 @@ func (a *API) Models() []ModelView {
|
||||
}
|
||||
measured[rep.Model] = max(measured[rep.Model], b)
|
||||
}
|
||||
var updates map[string]modelserve.UpdateInfo
|
||||
if a.serve != nil {
|
||||
updates = a.serve.Updates()
|
||||
}
|
||||
out := make([]ModelView, 0)
|
||||
for _, m := range a.ctl.Catalog().Models() {
|
||||
h := health[m.Name]
|
||||
@@ -605,6 +626,15 @@ func (a *API) Models() []ModelView {
|
||||
if len(m.Profiles) > 0 {
|
||||
v.MemoryGB, v.CPUCores = m.Profiles[0].MinMemoryGB, m.Profiles[0].CPUCores
|
||||
}
|
||||
if o := m.Origin; o != nil {
|
||||
v.Origin = &OriginView{Repo: o.Repo, Revision: o.Revision}
|
||||
if o.Previous != nil {
|
||||
v.Origin.Previous = o.Previous.Revision
|
||||
}
|
||||
}
|
||||
if u, ok := updates[m.Name]; ok {
|
||||
v.Update = &u
|
||||
}
|
||||
v.MeasuredMemoryGB = float64(measured[m.Name]) / (1 << 30)
|
||||
v.Warnings = reservationWarnings(m, v.MeasuredMemoryGB)
|
||||
out = append(out, v)
|
||||
@@ -1426,6 +1456,70 @@ func (a *API) job(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, job)
|
||||
}
|
||||
|
||||
func (a *API) listUpdates(w http.ResponseWriter, _ *http.Request) {
|
||||
out := make([]modelserve.UpdateInfo, 0)
|
||||
if a.serve != nil {
|
||||
for _, u := range a.serve.Updates() {
|
||||
out = append(out, u)
|
||||
}
|
||||
}
|
||||
slices.SortFunc(out, func(x, y modelserve.UpdateInfo) int { return strings.Compare(x.Model, y.Model) })
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// CheckRequest asks the registry about one model now, or every one.
|
||||
type CheckRequest struct {
|
||||
Model string `json:"model,omitempty"`
|
||||
}
|
||||
|
||||
func (a *API) checkUpdates(w http.ResponseWriter, r *http.Request) {
|
||||
if a.serve == nil {
|
||||
writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one")
|
||||
return
|
||||
}
|
||||
var req CheckRequest
|
||||
_ = json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req)
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Minute)
|
||||
defer cancel()
|
||||
if req.Model != "" {
|
||||
u, err := a.serve.CheckModel(ctx, req.Model)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, []modelserve.UpdateInfo{u})
|
||||
return
|
||||
}
|
||||
a.serve.CheckUpdates(ctx)
|
||||
a.listUpdates(w, r)
|
||||
}
|
||||
|
||||
func (a *API) updateModel(w http.ResponseWriter, r *http.Request) {
|
||||
if a.serve == nil {
|
||||
writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one")
|
||||
return
|
||||
}
|
||||
job, err := a.serve.StartUpdate(r.PathValue("name"), actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, job)
|
||||
}
|
||||
|
||||
func (a *API) rollbackModel(w http.ResponseWriter, r *http.Request) {
|
||||
if a.serve == nil {
|
||||
writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one")
|
||||
return
|
||||
}
|
||||
msg, err := a.serve.Rollback(r.Context(), r.PathValue("name"), actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"result": msg})
|
||||
}
|
||||
|
||||
// actorOf is who a change is recorded as made by: what the caller says, since
|
||||
// the admin API answers only this machine and trusts whoever reaches it. Kept
|
||||
// to a short printable name, so a claim cannot pass for more than one — no
|
||||
|
||||
Reference in new issue
Block a user