Track the registry's new weights for served models, and move a model to them or back

A model served from the registry records its origin: repository, commit and
each file's checksum; one served before is worked out from the mirror's
record of where it fetched the file. Every mirror.update_check (6h) the
control plane asks whether those files changed there. ctl model updates
lists it, ctl model update fetches beside the old files, checks, swaps and
replaces replicas one at a time, ctl model rollback goes back. The console
shows it on Models and offers the same.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 00:12:20 +02:00
1 parent a96b1a54b2
commit 44425193bd
15 files changed
+1393 -33

No files matched your search

+94
View File
@@ -121,6 +121,10 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /api/v1/restart", a.restart)
mux.HandleFunc("POST /api/v1/models/serve", a.serveModel)
mux.HandleFunc("GET /api/v1/jobs/{id}", a.job)
mux.HandleFunc("GET /api/v1/updates", a.listUpdates)
mux.HandleFunc("POST /api/v1/updates/check", a.checkUpdates)
mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel)
mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel)
mux.HandleFunc("GET /api/v1/requests/{id}", a.request)
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) {
@@ -336,6 +340,19 @@ type ModelView struct {
Adapters []catalog.Adapter `json:"adapters,omitempty"`
Preempted int `json:"preempted,omitempty"`
HeldFor string `json:"held_for,omitempty"`
// Origin is the registry repository its weights came from, and at which
// commit; Update what that repository had when it was last asked.
Origin *OriginView `json:"origin,omitempty"`
Update *modelserve.UpdateInfo `json:"update,omitempty"`
}
// OriginView is where a model's weights were published.
type OriginView struct {
Repo string `json:"repo"`
Revision string `json:"revision"`
// Previous is the commit the last update replaced, which it can go back
// to.
Previous string `json:"previous,omitempty"`
}
// ReplicaView is a replica as `ctl replicas` shows it.
@@ -586,6 +603,10 @@ func (a *API) Models() []ModelView {
}
measured[rep.Model] = max(measured[rep.Model], b)
}
var updates map[string]modelserve.UpdateInfo
if a.serve != nil {
updates = a.serve.Updates()
}
out := make([]ModelView, 0)
for _, m := range a.ctl.Catalog().Models() {
h := health[m.Name]
@@ -605,6 +626,15 @@ func (a *API) Models() []ModelView {
if len(m.Profiles) > 0 {
v.MemoryGB, v.CPUCores = m.Profiles[0].MinMemoryGB, m.Profiles[0].CPUCores
}
if o := m.Origin; o != nil {
v.Origin = &OriginView{Repo: o.Repo, Revision: o.Revision}
if o.Previous != nil {
v.Origin.Previous = o.Previous.Revision
}
}
if u, ok := updates[m.Name]; ok {
v.Update = &u
}
v.MeasuredMemoryGB = float64(measured[m.Name]) / (1 << 30)
v.Warnings = reservationWarnings(m, v.MeasuredMemoryGB)
out = append(out, v)
@@ -1426,6 +1456,70 @@ func (a *API) job(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, job)
}
func (a *API) listUpdates(w http.ResponseWriter, _ *http.Request) {
out := make([]modelserve.UpdateInfo, 0)
if a.serve != nil {
for _, u := range a.serve.Updates() {
out = append(out, u)
}
}
slices.SortFunc(out, func(x, y modelserve.UpdateInfo) int { return strings.Compare(x.Model, y.Model) })
writeJSON(w, http.StatusOK, out)
}
// CheckRequest asks the registry about one model now, or every one.
type CheckRequest struct {
Model string `json:"model,omitempty"`
}
func (a *API) checkUpdates(w http.ResponseWriter, r *http.Request) {
if a.serve == nil {
writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one")
return
}
var req CheckRequest
_ = json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req)
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Minute)
defer cancel()
if req.Model != "" {
u, err := a.serve.CheckModel(ctx, req.Model)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, []modelserve.UpdateInfo{u})
return
}
a.serve.CheckUpdates(ctx)
a.listUpdates(w, r)
}
func (a *API) updateModel(w http.ResponseWriter, r *http.Request) {
if a.serve == nil {
writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one")
return
}
job, err := a.serve.StartUpdate(r.PathValue("name"), actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusAccepted, job)
}
func (a *API) rollbackModel(w http.ResponseWriter, r *http.Request) {
if a.serve == nil {
writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one")
return
}
msg, err := a.serve.Rollback(r.Context(), r.PathValue("name"), actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"result": msg})
}
// actorOf is who a change is recorded as made by: what the caller says, since
// the admin API answers only this machine and trusts whoever reaches it. Kept
// to a short printable name, so a claim cannot pass for more than one — no
+45
View File
@@ -194,6 +194,51 @@ type Model struct {
// Capabilities is what this model can do beyond plain completion.
Capabilities Capabilities `json:"capabilities,omitempty" koanf:"capabilities"`
// Origin is the registry repository the weights came from, at which
// commit, and each file's checksum there. Written when a model is served
// from the registry; for one that was not, worked out from the mirror's
// record of where it fetched the file. It is what lets the control plane
// say that the repository has published different weights since, and
// move the model to them.
Origin *Origin `json:"origin,omitempty" koanf:"origin"`
}
// Origin is where a model's weights were published.
type Origin struct {
// Repo is the repository, owner/name.
Repo string `json:"repo" koanf:"repo"`
// Revision is the commit the files were fetched at.
Revision string `json:"revision" koanf:"revision"`
// Files are the weights fetched from it, with what the registry
// published as each one's sha256. A new commit that leaves them alone —
// a README edited — is not an update.
Files []OriginFile `json:"files,omitempty" koanf:"files"`
// Previous is what the last update replaced, kept so it can be gone back
// to while its files are still on the mirror.
Previous *PreviousOrigin `json:"previous,omitempty" koanf:"previous"`
}
// OriginFile is one weights file of an origin.
type OriginFile struct {
Path string `json:"path" koanf:"path"`
SHA256 string `json:"sha256,omitempty" koanf:"sha256"`
Bytes int64 `json:"bytes,omitempty" koanf:"bytes"`
}
// PreviousOrigin is a model's source before an update, and where it came from.
type PreviousOrigin struct {
Source string `json:"source" koanf:"source"`
Revision string `json:"revision" koanf:"revision"`
Files []OriginFile `json:"files,omitempty" koanf:"files"`
}
// ShortRevision is a commit as a person writes it.
func ShortRevision(rev string) string {
if len(rev) > 8 {
return rev[:8]
}
return rev
}
// Capabilities describes what a client may ask a model for.
+32 -2
View File
@@ -71,6 +71,12 @@ type Deps struct {
// ReadyTimeout bounds the wait for a replica: the node fetching the
// weights from the mirror and the engine loading them.
ReadyTimeout time.Duration
// For updates: the catalog in force, the service that changes it, and
// what replaces replicas. Without them updates are not offered.
Current func() []catalog.Model
Models Models
Restart func(ids ...string) error
}
// Request is what to serve.
@@ -116,7 +122,10 @@ type Step struct {
// Job is one model being served, and where it stands.
type Job struct {
ID string `json:"id"`
ID string `json:"id"`
// Kind is "update" for a model moved to new weights, empty for one
// served.
Kind string `json:"kind,omitempty"`
Request Request `json:"request"`
State string `json:"state"`
Error string `json:"error,omitempty"`
@@ -135,6 +144,8 @@ type Server struct {
mu sync.Mutex
jobs map[string]*Job
seq int
// updates is what the last checks of the registry found, by model.
updates map[string]UpdateInfo
}
// New builds a server.
@@ -151,7 +162,7 @@ func New(d Deps) *Server {
if d.Log == nil {
d.Log = slog.Default()
}
return &Server{deps: d, jobs: map[string]*Job{}}
return &Server{deps: d, jobs: map[string]*Job{}, updates: map[string]UpdateInfo{}}
}
var repoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
@@ -297,6 +308,7 @@ func (s *Server) run(j *Job) {
Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores,
}},
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)},
}
s.step(j, "catalog", Running, "adding "+name)
if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil {
@@ -388,8 +400,26 @@ func (s *Server) mirror(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSe
version := repo.ShortRevision()
s.step(j, "mirror", Running, "asking the mirror for "+set.Name)
var base string
// Files already there from this commit, checksums matching, are not
// fetched again: a model moved back and then forward again.
have := map[string]bool{}
if got, err := s.deps.Mirror.List(ctx); err == nil {
for _, st := range got.Models {
if st.Model == name && st.Version == version && st.State == mirror.Present {
for _, f := range set.Files {
if mirror.LastPathElement(f.Path) == st.File && (f.SHA256 == "" || strings.EqualFold(f.SHA256, st.SHA256)) {
have[st.File] = true
base = mirror.Entry{Model: name, Version: version, File: st.File}.URL(got.Base)
}
}
}
}
}
for _, f := range set.Files {
file := mirror.LastPathElement(f.Path)
if have[file] {
continue
}
resp, err := s.deps.Mirror.Fetch(ctx, mirror.FetchRequest{
Source: repo.DownloadURL("", f.Path), Model: name, Version: version, File: file,
})
+569
View File
@@ -0,0 +1,569 @@
package modelserve
import (
"context"
"errors"
"fmt"
"net/url"
"slices"
"strings"
"sync/atomic"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"github.com/prometheus/client_golang/prometheus"
)
// Updates: a repository publishes new weights under the same name — a fixed
// chat template, a better quantisation — and nothing told anyone. A model
// served from the registry now remembers where it came from; this asks the
// registry, every so often, whether the files it runs have changed there,
// and moves a model to them on request: fetched into the mirror beside the
// old ones, checked, swapped in, its replicas replaced one at a time, and
// the old files kept so it can go back.
// Models is the catalog as an update changes it.
type Models interface {
Declared(name string) bool
SetSource(ctx context.Context, name, source string, origin *catalog.Origin, by string) (catalog.Model, error)
SetOrigin(ctx context.Context, name string, origin *catalog.Origin) error
}
// UpdateInfo is what the registry has for a model against what it runs.
type UpdateInfo struct {
Model string `json:"model"`
Repo string `json:"repo,omitempty"`
// Current is the commit the model runs, Latest the repository's now.
Current string `json:"current,omitempty"`
Latest string `json:"latest,omitempty"`
// Changed are the model's files whose weights differ at Latest, and
// Missing those Latest no longer has at all.
Changed []string `json:"changed,omitempty"`
Missing []string `json:"missing,omitempty"`
// Bytes is what an update downloads.
Bytes int64 `json:"bytes,omitempty"`
// Available is true when the model can be moved to Latest.
Available bool `json:"available"`
// Previous is the commit an update replaced, which it can go back to.
Previous string `json:"previous,omitempty"`
CheckedAt time.Time `json:"checked_at"`
Err string `json:"error,omitempty"`
}
// place is where a model's weights sit on the mirror.
type place struct {
model, version string
files []mirror.Status
}
// placeOf finds a source on the mirror: a file, or a directory of shards.
func placeOf(source string, list mirror.ModelsResponse) (place, bool) {
base := strings.TrimSuffix(list.Base, "/") + "/"
if list.Base == "" || !strings.HasPrefix(source, base) {
return place{}, false
}
parts := strings.SplitN(strings.TrimPrefix(source, base), "/", 3)
if len(parts) < 2 {
return place{}, false
}
p := place{model: parts[0], version: parts[1]}
file := ""
if len(parts) == 3 {
file = parts[2]
}
for _, st := range list.Models {
if st.Model == p.model && st.Version == p.version && (file == "" || st.File == file) {
p.files = append(p.files, st)
}
}
return p, len(p.files) > 0
}
// parseResolve reads a registry download URL: base/owner/name/resolve/rev/path.
func parseResolve(u string) (repo, rev, path string, ok bool) {
pu, err := url.Parse(u)
if err != nil {
return "", "", "", false
}
i := strings.Index(pu.Path, "/resolve/")
if i < 0 {
return "", "", "", false
}
repo = strings.Trim(pu.Path[:i], "/")
if strings.Count(repo, "/") != 1 {
return "", "", "", false
}
rev, path, ok = strings.Cut(pu.Path[i+len("/resolve/"):], "/")
return repo, rev, path, ok && rev != "" && path != ""
}
// inferOrigin works a model's origin out from what the mirror recorded of
// where it fetched each file, for a model served before origins were kept.
func inferOrigin(p place) (*catalog.Origin, bool) {
var o catalog.Origin
for _, st := range p.files {
repo, rev, path, ok := parseResolve(st.Source)
if !ok || (o.Repo != "" && (repo != o.Repo || rev != o.Revision)) {
return nil, false
}
o.Repo, o.Revision = repo, rev
o.Files = append(o.Files, catalog.OriginFile{Path: path, SHA256: st.SHA256, Bytes: st.BytesDone})
}
return &o, o.Repo != ""
}
func originFiles(set hf.FileSet) []catalog.OriginFile {
out := make([]catalog.OriginFile, 0, len(set.Files))
for _, f := range set.Files {
out = append(out, catalog.OriginFile{Path: f.Path, SHA256: f.SHA256, Bytes: f.Bytes})
}
return out
}
// originOf is a model's origin: recorded, or worked out from the mirror and
// then recorded.
func (s *Server) originOf(ctx context.Context, m catalog.Model, list mirror.ModelsResponse) *catalog.Origin {
if m.Origin != nil && m.Origin.Repo != "" {
return m.Origin
}
p, ok := placeOf(m.Source, list)
if !ok {
return nil
}
o, ok := inferOrigin(p)
if !ok {
return nil
}
if s.deps.Models != nil {
if err := s.deps.Models.SetOrigin(ctx, m.Name, o); err != nil {
s.deps.Log.Debug("could not record a model's origin", "model", m.Name, "err", err)
}
}
return o
}
// compare says what the registry's repository has against an origin.
func compare(m catalog.Model, o *catalog.Origin, repo hf.Repo) UpdateInfo {
u := UpdateInfo{Model: m.Name, Repo: o.Repo, Current: o.Revision, Latest: repo.Revision, CheckedAt: time.Now()}
if o.Previous != nil {
u.Previous = o.Previous.Revision
}
if repo.Revision == o.Revision {
return u
}
for _, f := range o.Files {
i := slices.IndexFunc(repo.Files, func(g hf.File) bool { return g.Path == f.Path })
if i < 0 {
u.Missing = append(u.Missing, f.Path)
continue
}
g := repo.Files[i]
if (f.SHA256 != "" && g.SHA256 != "" && !strings.EqualFold(f.SHA256, g.SHA256)) ||
(f.SHA256 == "" && f.Bytes != 0 && f.Bytes != g.Bytes) {
u.Changed = append(u.Changed, f.Path)
u.Bytes += g.Bytes
}
}
u.Available = len(u.Changed) > 0 && len(u.Missing) == 0
return u
}
// CheckUpdates asks the registry about every model with an origin.
func (s *Server) CheckUpdates(ctx context.Context) {
if s.deps.Current == nil {
return
}
list, err := s.deps.Mirror.List(ctx)
if err != nil {
s.deps.Log.Warn("checking for model updates: the mirror", "err", err)
return
}
seen := map[string]bool{}
for _, m := range s.deps.Current() {
o := s.originOf(ctx, m, list)
if o == nil {
continue
}
seen[m.Name] = true
s.checkOne(ctx, m, o)
}
s.mu.Lock()
for name := range s.updates {
if !seen[name] {
delete(s.updates, name)
}
}
s.mu.Unlock()
}
func (s *Server) checkOne(ctx context.Context, m catalog.Model, o *catalog.Origin) UpdateInfo {
repo, err := s.deps.Registry.Repo(ctx, o.Repo)
u := UpdateInfo{Model: m.Name, Repo: o.Repo, Current: o.Revision, CheckedAt: time.Now()}
if err != nil {
u.Err = err.Error()
} else {
u = compare(m, o, repo)
if u.Available {
s.deps.Log.Info("a model's repository has new weights", "model", m.Name, "repo", o.Repo,
"current", catalog.ShortRevision(o.Revision), "latest", catalog.ShortRevision(repo.Revision))
}
}
s.mu.Lock()
s.updates[m.Name] = u
s.mu.Unlock()
return u
}
// CheckModel asks the registry about one model now.
func (s *Server) CheckModel(ctx context.Context, name string) (UpdateInfo, error) {
m, err := s.current(name)
if err != nil {
return UpdateInfo{}, err
}
list, err := s.deps.Mirror.List(ctx)
if err != nil {
return UpdateInfo{}, fmt.Errorf("the mirror: %w", err)
}
o := s.originOf(ctx, m, list)
if o == nil {
return UpdateInfo{}, fmt.Errorf("%s did not come from the registry through the mirror, so there is nothing to compare", name)
}
return s.checkOne(ctx, m, o), nil
}
// Updates is what the last checks found, by model.
func (s *Server) Updates() map[string]UpdateInfo {
s.mu.Lock()
defer s.mu.Unlock()
out := make(map[string]UpdateInfo, len(s.updates))
for k, v := range s.updates {
out[k] = v
}
return out
}
// WatchUpdates checks every so often until ctx ends; zero never checks.
func (s *Server) WatchUpdates(ctx context.Context, every time.Duration) {
if every <= 0 {
return
}
// A little after starting, when the catalog is in force and the mirror
// answering, rather than in the middle of starting.
wait := 2 * time.Minute
for {
select {
case <-ctx.Done():
return
case <-time.After(wait):
}
s.CheckUpdates(ctx)
wait = every
}
}
func (s *Server) current(name string) (catalog.Model, error) {
if s.deps.Current == nil {
return catalog.Model{}, errors.New("this control plane cannot read its catalog here")
}
for _, m := range s.deps.Current() {
if m.Name == name {
return m, nil
}
}
return catalog.Model{}, fmt.Errorf("no model called %s", name)
}
// StartUpdate moves a model to its repository's latest weights, as a job.
func (s *Server) StartUpdate(name, by string) (Job, error) {
if s.deps.Models == nil || s.deps.Restart == nil {
return Job{}, errors.New("updating needs the store, which this control plane lacks")
}
if _, err := s.current(name); err != nil {
return Job{}, err
}
if s.deps.Models.Declared(name) {
return Job{}, fmt.Errorf("%s is declared in server.yaml: serve the new weights there, by its source, "+
"or they would revert on the next restart", name)
}
s.mu.Lock()
for _, j := range s.jobs {
if j.Kind == "update" && j.Model == name && j.State == Running {
s.mu.Unlock()
return Job{}, fmt.Errorf("%s is already being updated: job %s", name, j.ID)
}
}
s.seq++
j := &Job{ID: fmt.Sprintf("update-%d-%d", time.Now().Unix(), s.seq), Kind: "update", Model: name,
Request: Request{Name: name, By: by}, State: Running, Started: time.Now()}
s.jobs[j.ID] = j
s.mu.Unlock()
go s.runUpdate(j)
return s.snapshot(j), nil
}
func (s *Server) runUpdate(j *Job) {
ctx := context.Background()
name := j.Model
// 1. The registry: which files, at which commit.
s.step(j, "registry", Running, "reading where "+name+" came from")
m, err := s.current(name)
if err != nil {
s.fail(j, "registry", err)
return
}
list, err := s.deps.Mirror.List(ctx)
if err != nil {
s.fail(j, "registry", fmt.Errorf("the mirror: %w", err))
return
}
o := s.originOf(ctx, m, list)
p, onMirror := placeOf(m.Source, list)
if o == nil || !onMirror {
s.fail(j, "registry", fmt.Errorf("%s did not come from the registry through the mirror", name))
return
}
repo, err := s.deps.Registry.Repo(ctx, o.Repo)
if err != nil {
s.fail(j, "registry", err)
return
}
s.mu.Lock()
j.Request.Repo = repo.ID
s.mu.Unlock()
u := compare(m, o, repo)
switch {
case repo.Revision == o.Revision:
s.fail(j, "registry", fmt.Errorf("%s already runs %s's latest commit, %s", name, repo.ID, catalog.ShortRevision(repo.Revision)))
return
case len(u.Missing) > 0:
s.fail(j, "registry", fmt.Errorf("%s at %s no longer has %s; choose one of its files and serve it under a new name",
repo.ID, repo.ShortRevision(), strings.Join(u.Missing, ", ")))
return
case len(u.Changed) == 0:
s.fail(j, "registry", fmt.Errorf("%s has a newer commit, %s, but the files %s runs are the same in it",
repo.ID, repo.ShortRevision(), name))
return
case repo.Gated.Is():
s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", repo.ID))
return
}
set := hf.FileSet{}
for _, f := range o.Files {
g := repo.Files[slices.IndexFunc(repo.Files, func(g hf.File) bool { return g.Path == f.Path })]
set.Files = append(set.Files, g)
set.Bytes += g.Bytes
}
set.Name, set.Shards = set.Files[0].Path, len(set.Files)
s.step(j, "registry", Done, fmt.Sprintf("%s: %s → %s, %d of %d file(s) changed, %.1f GB",
repo.ID, catalog.ShortRevision(o.Revision), repo.ShortRevision(), len(u.Changed), len(o.Files), gb(set.Bytes)))
// 2. The mirror, beside the files in use.
source, err := s.mirror(ctx, j, repo, set, p.model)
if err != nil {
s.fail(j, "mirror", err)
return
}
// 3. Whether the nodes load it, and whether it still fits what is
// reserved for it.
s.step(j, "check", Running, "reading its header from the mirror")
detail := "a set of shards; not read"
if !strings.HasSuffix(source, "/") {
v := gguf.Check(ctx, source, s.deps.LlamaVersion)
if !v.Servable {
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason))
return
}
detail = orUnknown(v.Header.Architecture)
if len(m.Profiles) > 0 {
ctxPool := 16384
for i, a := range m.Profiles[0].Args {
if a == "--ctx-size" && i+1 < len(m.Profiles[0].Args) {
fmt.Sscan(m.Profiles[0].Args[i+1], &ctxPool)
}
}
need := int((v.Header.MemoryBytes(set.Bytes, ctxPool) + 1<<30 - 1) >> 30)
if have := m.Profiles[0].MinMemoryGB; need > have && have > 0 {
detail += fmt.Sprintf("; needs about %d GB and %d is reserved: raise it if replicas fail", need, have)
}
}
}
s.step(j, "check", Done, detail)
// 4. The catalog: the new source, and the old one kept to go back to.
s.step(j, "catalog", Running, "moving "+name+" to "+repo.ShortRevision())
next := &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set),
Previous: &catalog.PreviousOrigin{Source: m.Source, Revision: o.Revision, Files: o.Files}}
if _, err := s.deps.Models.SetSource(ctx, name, source, next, j.Request.By); err != nil {
s.fail(j, "catalog", err)
return
}
s.step(j, "catalog", Done, "the catalog names "+source)
// 5. The replicas, replaced one at a time.
if err := s.replace(ctx, j, name); err != nil {
s.fail(j, "replicas", fmt.Errorf("%w. `llmgw ctl model rollback %s` goes back to %s", err, name,
catalog.ShortRevision(o.Revision)))
return
}
s.mu.Lock()
s.updates[name] = UpdateInfo{Model: name, Repo: repo.ID, Current: repo.Revision, Latest: repo.Revision,
Previous: o.Revision, CheckedAt: time.Now()}
s.mu.Unlock()
s.finish(j)
}
// replace restarts a model's replicas onto the catalog in force and waits
// for the new ones, then asks one to answer.
func (s *Server) replace(ctx context.Context, j *Job, name string) error {
var old []string
running := 0
for _, r := range s.deps.Fleet.Replicas() {
if r.Model != name {
continue
}
running++
if r.State == reconciler.Ready {
old = append(old, r.ID)
}
}
if running == 0 {
s.step(j, "replicas", Skipped, "none running; the next to start loads the new weights")
return nil
}
if len(old) == 0 {
s.step(j, "replicas", Skipped, "none ready; those starting load the new weights when they next start")
return nil
}
if err := s.deps.Restart(old...); err != nil {
return err
}
s.step(j, "replicas", Running, fmt.Sprintf("replacing %d, one at a time", len(old)))
deadline := time.Now().Add(s.deps.ReadyTimeout)
for {
left, ready := 0, []reconciler.Replica{}
for _, r := range s.deps.Fleet.Replicas() {
if r.Model != name {
continue
}
if slices.Contains(old, r.ID) {
left++
} else if r.State == reconciler.Ready {
ready = append(ready, r)
}
}
for _, h := range s.deps.Fleet.ModelHealth() {
if h.Model == name && h.Parked {
return fmt.Errorf("the new weights failed to start %d times: %s", h.Failures, h.Reason)
}
}
if left == 0 && len(ready) > 0 {
s.step(j, "replicas", Done, fmt.Sprintf("%d replaced", len(old)))
s.step(j, "smoke test", Running, "asking "+ready[0].ID+" to say hello")
reply, tps, err := s.smoke(ctx, ready[0], name)
if err != nil {
s.step(j, "smoke test", Failed, err.Error())
return err
}
s.mu.Lock()
j.Reply, j.TokensPerSecond = reply, tps
s.mu.Unlock()
s.step(j, "smoke test", Done, fmt.Sprintf("answered at %.1f tokens/s: %q", tps, reply))
return nil
}
s.step(j, "replicas", Running, fmt.Sprintf("%d of %d replaced, %d new ready", len(old)-left, len(old), len(ready)))
if time.Now().After(deadline) {
return fmt.Errorf("the replicas were not replaced after %s", s.deps.ReadyTimeout)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(s.deps.Poll):
}
}
}
// Rollback moves a model back to what its last update replaced, if those
// files are still on the mirror, and restarts its replicas onto them. The
// update is kept as the one to go forward to again.
func (s *Server) Rollback(ctx context.Context, name, by string) (string, error) {
if s.deps.Models == nil || s.deps.Restart == nil {
return "", errors.New("going back needs the store, which this control plane lacks")
}
m, err := s.current(name)
if err != nil {
return "", err
}
if m.Origin == nil || m.Origin.Previous == nil {
return "", fmt.Errorf("%s has not been updated, so there is nothing to go back to", name)
}
prev := m.Origin.Previous
list, err := s.deps.Mirror.List(ctx)
if err != nil {
return "", fmt.Errorf("the mirror: %w", err)
}
if _, ok := placeOf(prev.Source, list); !ok {
return "", fmt.Errorf("the files of %s are no longer on the mirror", catalog.ShortRevision(prev.Revision))
}
back := &catalog.Origin{Repo: m.Origin.Repo, Revision: prev.Revision, Files: prev.Files,
Previous: &catalog.PreviousOrigin{Source: m.Source, Revision: m.Origin.Revision, Files: m.Origin.Files}}
if _, err := s.deps.Models.SetSource(ctx, name, prev.Source, back, by); err != nil {
return "", err
}
var ids []string
for _, r := range s.deps.Fleet.Replicas() {
if r.Model == name && r.State == reconciler.Ready {
ids = append(ids, r.ID)
}
}
msg := fmt.Sprintf("%s runs %s again", name, catalog.ShortRevision(prev.Revision))
if len(ids) > 0 {
if err := s.deps.Restart(ids...); err != nil {
return "", err
}
msg += fmt.Sprintf("; %d replica(s) are being replaced", len(ids))
}
s.mu.Lock()
if u, ok := s.updates[name]; ok {
u.Current, u.Previous = prev.Revision, m.Origin.Revision
u.Available = u.Latest != "" && u.Latest != prev.Revision
s.updates[name] = u
}
s.mu.Unlock()
return msg, nil
}
// UpdateCollector publishes llmgw_model_update_available, 1 for each model
// whose repository has new weights. Registered when the metrics are, before
// there is a server to read: Set gives it one.
type UpdateCollector struct{ s atomic.Pointer[Server] }
var updateDesc = prometheus.NewDesc("llmgw_model_update_available",
"1 when the registry repository a model was served from has published different weights for its files.",
[]string{"model", "repo"}, nil)
// Set gives the collector the server whose checks it reports.
func (c *UpdateCollector) Set(s *Server) { c.s.Store(s) }
// Describe implements prometheus.Collector.
func (c *UpdateCollector) Describe(ch chan<- *prometheus.Desc) { ch <- updateDesc }
// Collect implements prometheus.Collector.
func (c *UpdateCollector) Collect(ch chan<- prometheus.Metric) {
s := c.s.Load()
if s == nil {
return
}
for _, u := range s.Updates() {
v := 0.0
if u.Available {
v = 1
}
ch <- prometheus.MustNewConstMetric(updateDesc, prometheus.GaugeValue, v, u.Model, u.Repo)
}
}
+264
View File
@@ -0,0 +1,264 @@
package modelserve
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
)
const (
oldRev = "abc123456789aaaa"
newRev = "def987654321bbbb"
)
type movingRegistry struct {
mu sync.Mutex
repo hf.Repo
}
func (r *movingRegistry) Repo(context.Context, string) (hf.Repo, error) {
r.mu.Lock()
defer r.mu.Unlock()
return r.repo, nil
}
// updMirror is a mirror that records where it fetched each file from.
type updMirror struct {
mu sync.Mutex
base string
held []mirror.Status
sums map[string]string
}
func (m *updMirror) Fetch(_ context.Context, r mirror.FetchRequest) (mirror.FetchResponse, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.held = append(m.held, mirror.Status{Model: r.Model, Version: r.Version, File: r.File, Source: r.Source,
State: mirror.Present, BytesDone: 1 << 30, SHA256: m.sums[r.Version+"/"+r.File]})
return mirror.FetchResponse{URL: mirror.Entry{Model: r.Model, Version: r.Version, File: r.File}.URL(m.base)}, nil
}
func (m *updMirror) List(context.Context) (mirror.ModelsResponse, error) {
m.mu.Lock()
defer m.mu.Unlock()
return mirror.ModelsResponse{Base: m.base, Models: append([]mirror.Status{}, m.held...)}, nil
}
type updModels struct {
mu sync.Mutex
models map[string]catalog.Model
declared map[string]bool
}
func (c *updModels) Declared(n string) bool { return c.declared[n] }
func (c *updModels) SetSource(_ context.Context, n, src string, o *catalog.Origin, _ string) (catalog.Model, error) {
c.mu.Lock()
defer c.mu.Unlock()
m := c.models[n]
m.Source, m.Origin = src, o
c.models[n] = m
return m, nil
}
func (c *updModels) SetOrigin(_ context.Context, n string, o *catalog.Origin) error {
c.mu.Lock()
defer c.mu.Unlock()
m := c.models[n]
m.Origin = o
c.models[n] = m
return nil
}
func (c *updModels) all() []catalog.Model {
c.mu.Lock()
defer c.mu.Unlock()
var out []catalog.Model
for _, m := range c.models {
out = append(out, m)
}
return out
}
// restartFleet replaces a restarted replica with a new one, ready.
type restartFleet struct {
mu sync.Mutex
reps []reconciler.Replica
engine string
seq int
}
func (f *restartFleet) Replicas() []reconciler.Replica {
f.mu.Lock()
defer f.mu.Unlock()
return append([]reconciler.Replica{}, f.reps...)
}
func (f *restartFleet) ModelHealth() []reconciler.ModelHealth { return nil }
func (f *restartFleet) EngineKey(string) string { return "" }
func (f *restartFleet) Restart(ids ...string) error {
f.mu.Lock()
defer f.mu.Unlock()
for _, id := range ids {
for i, r := range f.reps {
if r.ID == id {
f.seq++
f.reps[i] = reconciler.Replica{ID: r.Model + "-new" + string(rune('0'+f.seq)), Model: r.Model,
State: reconciler.Ready, Endpoint: f.engine}
}
}
}
return nil
}
func updSetup(t *testing.T) (*Server, *updMirror, *updModels, *movingRegistry, *restartFleet) {
t.Helper()
header := ggufHeader()
files := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(header) }))
t.Cleanup(files.Close)
engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"Hi"}}],"usage":{"completion_tokens":1}}`))
}))
t.Cleanup(engine.Close)
mir := &updMirror{base: files.URL, sums: map[string]string{
"abc12345/q4.gguf": "aaaa", "def98765/q4.gguf": "bbbb"}}
// What was fetched before origins were kept: the mirror alone says where from.
_, _ = mir.Fetch(context.Background(), mirror.FetchRequest{Model: "qwen", Version: "abc12345", File: "q4.gguf",
Source: "https://huggingface.co/o/r/resolve/" + oldRev + "/q4.gguf"})
models := &updModels{models: map[string]catalog.Model{"qwen": {Name: "qwen",
Source: files.URL + "/qwen/abc12345/q4.gguf",
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 8}}}}, declared: map[string]bool{}}
reg := &movingRegistry{repo: hf.Repo{ID: "o/r", Revision: newRev, Files: []hf.File{
{Path: "q4.gguf", SHA256: "bbbb", Bytes: 2 << 30}, {Path: "README.md", Bytes: 10}}}}
fleet := &restartFleet{engine: engine.URL, reps: []reconciler.Replica{
{ID: "qwen-1", Model: "qwen", State: reconciler.Ready, Endpoint: engine.URL},
{ID: "qwen-2", Model: "qwen", State: reconciler.Ready, Endpoint: engine.URL}}}
s := New(Deps{Registry: reg, Mirror: mir, Catalog: &fakeCatalog{}, Fleet: fleet,
Current: models.all, Models: models, Restart: fleet.Restart,
Poll: 10 * time.Millisecond, ReadyTimeout: 5 * time.Second})
return s, mir, models, reg, fleet
}
func TestNewWeightsInTheRepositoryAreFoundForAModelServedBeforeOriginsWereKept(t *testing.T) {
s, _, models, _, _ := updSetup(t)
s.CheckUpdates(context.Background())
u := s.Updates()["qwen"]
if !u.Available || u.Repo != "o/r" || u.Current != oldRev || u.Latest != newRev ||
len(u.Changed) != 1 || u.Changed[0] != "q4.gguf" || u.Bytes != 2<<30 {
t.Fatalf("%+v", u)
}
if o := models.models["qwen"].Origin; o == nil || o.Repo != "o/r" || o.Files[0].SHA256 != "aaaa" {
t.Errorf("the origin worked out was not recorded: %+v", o)
}
}
func TestANewCommitThatLeavesTheWeightsAloneIsNotAnUpdate(t *testing.T) {
s, mir, _, _, _ := updSetup(t)
mir.held[0].SHA256 = "bbbb"
u, err := s.CheckModel(context.Background(), "qwen")
if err != nil || u.Available || u.Latest != newRev || updateSays(u) != "same" {
t.Fatalf("%+v %v", u, err)
}
}
func updateSays(u UpdateInfo) string {
if !u.Available && u.Latest != u.Current && len(u.Missing) == 0 {
return "same"
}
return ""
}
func TestAnUpdateFetchesBesideTheOldReplacesTheReplicasAndCanGoBack(t *testing.T) {
s, mir, models, _, fleet := updSetup(t)
oldSource := models.models["qwen"].Source
job, err := s.StartUpdate("qwen", "tester")
if err != nil {
t.Fatal(err)
}
j := wait(t, s, job.ID)
if j.State != Done {
t.Fatalf("%s: %+v", j.Error, j.Steps)
}
m := models.models["qwen"]
if !strings.HasSuffix(m.Source, "/qwen/def98765/q4.gguf") || m.Origin.Revision != newRev ||
m.Origin.Previous == nil || m.Origin.Previous.Source != oldSource {
t.Fatalf("catalog: %s %+v", m.Source, m.Origin)
}
if len(mir.held) != 2 {
t.Errorf("the old files did not stay on the mirror beside the new: %+v", mir.held)
}
for _, r := range fleet.Replicas() {
if !strings.Contains(r.ID, "new") {
t.Errorf("%s was not replaced", r.ID)
}
}
if u := s.Updates()["qwen"]; u.Available || u.Current != newRev {
t.Errorf("still offered: %+v", u)
}
// And back.
if _, err := s.Rollback(context.Background(), "qwen", "tester"); err != nil {
t.Fatal(err)
}
m = models.models["qwen"]
if m.Source != oldSource || m.Origin.Revision != oldRev || m.Origin.Previous.Revision != newRev {
t.Fatalf("rollback: %s %+v", m.Source, m.Origin)
}
if u := s.Updates()["qwen"]; !u.Available {
t.Errorf("the update it went back from is not offered again: %+v", u)
}
// And forward again, from files already on the mirror.
job, err = s.StartUpdate("qwen", "tester")
if err != nil {
t.Fatal(err)
}
if j := wait(t, s, job.ID); j.State != Done || models.models["qwen"].Origin.Revision != newRev {
t.Fatalf("%s: %+v", j.Error, j.Steps)
}
// Then there is nothing newer: refused, saying why.
job, _ = s.StartUpdate("qwen", "tester")
if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "already runs") {
t.Fatalf("%+v", j)
}
}
func TestAModelFromConfigurationIsNotUpdatedBehindItsBack(t *testing.T) {
s, _, models, _, _ := updSetup(t)
models.declared["qwen"] = true
if _, err := s.StartUpdate("qwen", "t"); err == nil || !strings.Contains(err.Error(), "server.yaml") {
t.Fatalf("%v", err)
}
}
func TestAnUpdateToFilesTheRepositoryDroppedIsRefused(t *testing.T) {
s, _, _, reg, _ := updSetup(t)
reg.repo.Files = []hf.File{{Path: "q4-v2.gguf", SHA256: "cccc"}}
job, _ := s.StartUpdate("qwen", "t")
if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "no longer has q4.gguf") {
t.Fatalf("%+v", j)
}
}
func TestRegistryDownloadURLsAreRead(t *testing.T) {
for u, want := range map[string][3]string{
"https://huggingface.co/bartowski/Qwen-GGUF/resolve/abc/sub/f.gguf": {"bartowski/Qwen-GGUF", "abc", "sub/f.gguf"},
"https://hf.internal/mirror/x/resolve/main/f.gguf": {"mirror/x", "main", "f.gguf"},
} {
repo, rev, path, ok := parseResolve(u)
if !ok || [3]string{repo, rev, path} != want {
t.Errorf("%s: %s %s %s %v", u, repo, rev, path, ok)
}
}
for _, u := range []string{"https://example.com/f.gguf", "https://huggingface.co/a/b/c/resolve/x/f", ":"} {
if _, _, _, ok := parseResolve(u); ok {
t.Errorf("%s was read", u)
}
}
}
+32
View File
@@ -288,6 +288,38 @@ func (s *Service) SetAdapters(ctx context.Context, name string, adapters []catal
return final, err
}
// SetSource moves a model added at runtime to other weights, recording where
// they came from. Running replicas keep what they loaded until they are
// replaced; the caller restarts them.
func (s *Service) SetSource(ctx context.Context, name, source string, origin *catalog.Origin, by string) (catalog.Model, error) {
final, err := s.update(ctx, name, by, "change its source", func(m *catalog.Model) error {
m.Source, m.Origin = source, origin
return nil
})
if err == nil {
rev := ""
if origin != nil {
rev = origin.Revision
}
s.log.Info("model source set", "model", name, "source", source, "revision", rev, "by", by)
}
return final, err
}
// SetOrigin records where a model's weights came from without changing them:
// for a model added before origins were kept. A model from configuration is
// left alone, its origin worked out again when it is needed.
func (s *Service) SetOrigin(ctx context.Context, name string, origin *catalog.Origin) error {
if s.Declared(name) {
return nil
}
_, err := s.update(ctx, name, "llmgw", "record its origin", func(m *catalog.Model) error {
m.Origin = origin
return nil
})
return err
}
// update changes one model added at runtime and keeps the change.
//
// Built before anything is written, like Add: catalog.New is what applies the