Files
llmgw/internal/control/modelserve/updates.go
T
Ilker ManapandClaude Opus 5.5 44425193bd Track the registry's new weights for served models, and move a model to them or back
A model served from the registry records its origin: repository, commit and
each file's checksum; one served before is worked out from the mirror's
record of where it fetched the file. Every mirror.update_check (6h) the
control plane asks whether those files changed there. ctl model updates
lists it, ctl model update fetches beside the old files, checks, swaps and
replaces replicas one at a time, ctl model rollback goes back. The console
shows it on Models and offers the same.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-04 00:12:20 +02:00

570 lines
18 KiB
Go

package modelserve
import (
"context"
"errors"
"fmt"
"net/url"
"slices"
"strings"
"sync/atomic"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"github.com/prometheus/client_golang/prometheus"
)
// Updates: a repository publishes new weights under the same name — a fixed
// chat template, a better quantisation — and nothing told anyone. A model
// served from the registry now remembers where it came from; this asks the
// registry, every so often, whether the files it runs have changed there,
// and moves a model to them on request: fetched into the mirror beside the
// old ones, checked, swapped in, its replicas replaced one at a time, and
// the old files kept so it can go back.
// Models is the catalog as an update changes it.
type Models interface {
Declared(name string) bool
SetSource(ctx context.Context, name, source string, origin *catalog.Origin, by string) (catalog.Model, error)
SetOrigin(ctx context.Context, name string, origin *catalog.Origin) error
}
// UpdateInfo is what the registry has for a model against what it runs.
type UpdateInfo struct {
Model string `json:"model"`
Repo string `json:"repo,omitempty"`
// Current is the commit the model runs, Latest the repository's now.
Current string `json:"current,omitempty"`
Latest string `json:"latest,omitempty"`
// Changed are the model's files whose weights differ at Latest, and
// Missing those Latest no longer has at all.
Changed []string `json:"changed,omitempty"`
Missing []string `json:"missing,omitempty"`
// Bytes is what an update downloads.
Bytes int64 `json:"bytes,omitempty"`
// Available is true when the model can be moved to Latest.
Available bool `json:"available"`
// Previous is the commit an update replaced, which it can go back to.
Previous string `json:"previous,omitempty"`
CheckedAt time.Time `json:"checked_at"`
Err string `json:"error,omitempty"`
}
// place is where a model's weights sit on the mirror.
type place struct {
model, version string
files []mirror.Status
}
// placeOf finds a source on the mirror: a file, or a directory of shards.
func placeOf(source string, list mirror.ModelsResponse) (place, bool) {
base := strings.TrimSuffix(list.Base, "/") + "/"
if list.Base == "" || !strings.HasPrefix(source, base) {
return place{}, false
}
parts := strings.SplitN(strings.TrimPrefix(source, base), "/", 3)
if len(parts) < 2 {
return place{}, false
}
p := place{model: parts[0], version: parts[1]}
file := ""
if len(parts) == 3 {
file = parts[2]
}
for _, st := range list.Models {
if st.Model == p.model && st.Version == p.version && (file == "" || st.File == file) {
p.files = append(p.files, st)
}
}
return p, len(p.files) > 0
}
// parseResolve reads a registry download URL: base/owner/name/resolve/rev/path.
func parseResolve(u string) (repo, rev, path string, ok bool) {
pu, err := url.Parse(u)
if err != nil {
return "", "", "", false
}
i := strings.Index(pu.Path, "/resolve/")
if i < 0 {
return "", "", "", false
}
repo = strings.Trim(pu.Path[:i], "/")
if strings.Count(repo, "/") != 1 {
return "", "", "", false
}
rev, path, ok = strings.Cut(pu.Path[i+len("/resolve/"):], "/")
return repo, rev, path, ok && rev != "" && path != ""
}
// inferOrigin works a model's origin out from what the mirror recorded of
// where it fetched each file, for a model served before origins were kept.
func inferOrigin(p place) (*catalog.Origin, bool) {
var o catalog.Origin
for _, st := range p.files {
repo, rev, path, ok := parseResolve(st.Source)
if !ok || (o.Repo != "" && (repo != o.Repo || rev != o.Revision)) {
return nil, false
}
o.Repo, o.Revision = repo, rev
o.Files = append(o.Files, catalog.OriginFile{Path: path, SHA256: st.SHA256, Bytes: st.BytesDone})
}
return &o, o.Repo != ""
}
func originFiles(set hf.FileSet) []catalog.OriginFile {
out := make([]catalog.OriginFile, 0, len(set.Files))
for _, f := range set.Files {
out = append(out, catalog.OriginFile{Path: f.Path, SHA256: f.SHA256, Bytes: f.Bytes})
}
return out
}
// originOf is a model's origin: recorded, or worked out from the mirror and
// then recorded.
func (s *Server) originOf(ctx context.Context, m catalog.Model, list mirror.ModelsResponse) *catalog.Origin {
if m.Origin != nil && m.Origin.Repo != "" {
return m.Origin
}
p, ok := placeOf(m.Source, list)
if !ok {
return nil
}
o, ok := inferOrigin(p)
if !ok {
return nil
}
if s.deps.Models != nil {
if err := s.deps.Models.SetOrigin(ctx, m.Name, o); err != nil {
s.deps.Log.Debug("could not record a model's origin", "model", m.Name, "err", err)
}
}
return o
}
// compare says what the registry's repository has against an origin.
func compare(m catalog.Model, o *catalog.Origin, repo hf.Repo) UpdateInfo {
u := UpdateInfo{Model: m.Name, Repo: o.Repo, Current: o.Revision, Latest: repo.Revision, CheckedAt: time.Now()}
if o.Previous != nil {
u.Previous = o.Previous.Revision
}
if repo.Revision == o.Revision {
return u
}
for _, f := range o.Files {
i := slices.IndexFunc(repo.Files, func(g hf.File) bool { return g.Path == f.Path })
if i < 0 {
u.Missing = append(u.Missing, f.Path)
continue
}
g := repo.Files[i]
if (f.SHA256 != "" && g.SHA256 != "" && !strings.EqualFold(f.SHA256, g.SHA256)) ||
(f.SHA256 == "" && f.Bytes != 0 && f.Bytes != g.Bytes) {
u.Changed = append(u.Changed, f.Path)
u.Bytes += g.Bytes
}
}
u.Available = len(u.Changed) > 0 && len(u.Missing) == 0
return u
}
// CheckUpdates asks the registry about every model with an origin.
func (s *Server) CheckUpdates(ctx context.Context) {
if s.deps.Current == nil {
return
}
list, err := s.deps.Mirror.List(ctx)
if err != nil {
s.deps.Log.Warn("checking for model updates: the mirror", "err", err)
return
}
seen := map[string]bool{}
for _, m := range s.deps.Current() {
o := s.originOf(ctx, m, list)
if o == nil {
continue
}
seen[m.Name] = true
s.checkOne(ctx, m, o)
}
s.mu.Lock()
for name := range s.updates {
if !seen[name] {
delete(s.updates, name)
}
}
s.mu.Unlock()
}
func (s *Server) checkOne(ctx context.Context, m catalog.Model, o *catalog.Origin) UpdateInfo {
repo, err := s.deps.Registry.Repo(ctx, o.Repo)
u := UpdateInfo{Model: m.Name, Repo: o.Repo, Current: o.Revision, CheckedAt: time.Now()}
if err != nil {
u.Err = err.Error()
} else {
u = compare(m, o, repo)
if u.Available {
s.deps.Log.Info("a model's repository has new weights", "model", m.Name, "repo", o.Repo,
"current", catalog.ShortRevision(o.Revision), "latest", catalog.ShortRevision(repo.Revision))
}
}
s.mu.Lock()
s.updates[m.Name] = u
s.mu.Unlock()
return u
}
// CheckModel asks the registry about one model now.
func (s *Server) CheckModel(ctx context.Context, name string) (UpdateInfo, error) {
m, err := s.current(name)
if err != nil {
return UpdateInfo{}, err
}
list, err := s.deps.Mirror.List(ctx)
if err != nil {
return UpdateInfo{}, fmt.Errorf("the mirror: %w", err)
}
o := s.originOf(ctx, m, list)
if o == nil {
return UpdateInfo{}, fmt.Errorf("%s did not come from the registry through the mirror, so there is nothing to compare", name)
}
return s.checkOne(ctx, m, o), nil
}
// Updates is what the last checks found, by model.
func (s *Server) Updates() map[string]UpdateInfo {
s.mu.Lock()
defer s.mu.Unlock()
out := make(map[string]UpdateInfo, len(s.updates))
for k, v := range s.updates {
out[k] = v
}
return out
}
// WatchUpdates checks every so often until ctx ends; zero never checks.
func (s *Server) WatchUpdates(ctx context.Context, every time.Duration) {
if every <= 0 {
return
}
// A little after starting, when the catalog is in force and the mirror
// answering, rather than in the middle of starting.
wait := 2 * time.Minute
for {
select {
case <-ctx.Done():
return
case <-time.After(wait):
}
s.CheckUpdates(ctx)
wait = every
}
}
func (s *Server) current(name string) (catalog.Model, error) {
if s.deps.Current == nil {
return catalog.Model{}, errors.New("this control plane cannot read its catalog here")
}
for _, m := range s.deps.Current() {
if m.Name == name {
return m, nil
}
}
return catalog.Model{}, fmt.Errorf("no model called %s", name)
}
// StartUpdate moves a model to its repository's latest weights, as a job.
func (s *Server) StartUpdate(name, by string) (Job, error) {
if s.deps.Models == nil || s.deps.Restart == nil {
return Job{}, errors.New("updating needs the store, which this control plane lacks")
}
if _, err := s.current(name); err != nil {
return Job{}, err
}
if s.deps.Models.Declared(name) {
return Job{}, fmt.Errorf("%s is declared in server.yaml: serve the new weights there, by its source, "+
"or they would revert on the next restart", name)
}
s.mu.Lock()
for _, j := range s.jobs {
if j.Kind == "update" && j.Model == name && j.State == Running {
s.mu.Unlock()
return Job{}, fmt.Errorf("%s is already being updated: job %s", name, j.ID)
}
}
s.seq++
j := &Job{ID: fmt.Sprintf("update-%d-%d", time.Now().Unix(), s.seq), Kind: "update", Model: name,
Request: Request{Name: name, By: by}, State: Running, Started: time.Now()}
s.jobs[j.ID] = j
s.mu.Unlock()
go s.runUpdate(j)
return s.snapshot(j), nil
}
func (s *Server) runUpdate(j *Job) {
ctx := context.Background()
name := j.Model
// 1. The registry: which files, at which commit.
s.step(j, "registry", Running, "reading where "+name+" came from")
m, err := s.current(name)
if err != nil {
s.fail(j, "registry", err)
return
}
list, err := s.deps.Mirror.List(ctx)
if err != nil {
s.fail(j, "registry", fmt.Errorf("the mirror: %w", err))
return
}
o := s.originOf(ctx, m, list)
p, onMirror := placeOf(m.Source, list)
if o == nil || !onMirror {
s.fail(j, "registry", fmt.Errorf("%s did not come from the registry through the mirror", name))
return
}
repo, err := s.deps.Registry.Repo(ctx, o.Repo)
if err != nil {
s.fail(j, "registry", err)
return
}
s.mu.Lock()
j.Request.Repo = repo.ID
s.mu.Unlock()
u := compare(m, o, repo)
switch {
case repo.Revision == o.Revision:
s.fail(j, "registry", fmt.Errorf("%s already runs %s's latest commit, %s", name, repo.ID, catalog.ShortRevision(repo.Revision)))
return
case len(u.Missing) > 0:
s.fail(j, "registry", fmt.Errorf("%s at %s no longer has %s; choose one of its files and serve it under a new name",
repo.ID, repo.ShortRevision(), strings.Join(u.Missing, ", ")))
return
case len(u.Changed) == 0:
s.fail(j, "registry", fmt.Errorf("%s has a newer commit, %s, but the files %s runs are the same in it",
repo.ID, repo.ShortRevision(), name))
return
case repo.Gated.Is():
s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", repo.ID))
return
}
set := hf.FileSet{}
for _, f := range o.Files {
g := repo.Files[slices.IndexFunc(repo.Files, func(g hf.File) bool { return g.Path == f.Path })]
set.Files = append(set.Files, g)
set.Bytes += g.Bytes
}
set.Name, set.Shards = set.Files[0].Path, len(set.Files)
s.step(j, "registry", Done, fmt.Sprintf("%s: %s → %s, %d of %d file(s) changed, %.1f GB",
repo.ID, catalog.ShortRevision(o.Revision), repo.ShortRevision(), len(u.Changed), len(o.Files), gb(set.Bytes)))
// 2. The mirror, beside the files in use.
source, err := s.mirror(ctx, j, repo, set, p.model)
if err != nil {
s.fail(j, "mirror", err)
return
}
// 3. Whether the nodes load it, and whether it still fits what is
// reserved for it.
s.step(j, "check", Running, "reading its header from the mirror")
detail := "a set of shards; not read"
if !strings.HasSuffix(source, "/") {
v := gguf.Check(ctx, source, s.deps.LlamaVersion)
if !v.Servable {
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason))
return
}
detail = orUnknown(v.Header.Architecture)
if len(m.Profiles) > 0 {
ctxPool := 16384
for i, a := range m.Profiles[0].Args {
if a == "--ctx-size" && i+1 < len(m.Profiles[0].Args) {
fmt.Sscan(m.Profiles[0].Args[i+1], &ctxPool)
}
}
need := int((v.Header.MemoryBytes(set.Bytes, ctxPool) + 1<<30 - 1) >> 30)
if have := m.Profiles[0].MinMemoryGB; need > have && have > 0 {
detail += fmt.Sprintf("; needs about %d GB and %d is reserved: raise it if replicas fail", need, have)
}
}
}
s.step(j, "check", Done, detail)
// 4. The catalog: the new source, and the old one kept to go back to.
s.step(j, "catalog", Running, "moving "+name+" to "+repo.ShortRevision())
next := &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set),
Previous: &catalog.PreviousOrigin{Source: m.Source, Revision: o.Revision, Files: o.Files}}
if _, err := s.deps.Models.SetSource(ctx, name, source, next, j.Request.By); err != nil {
s.fail(j, "catalog", err)
return
}
s.step(j, "catalog", Done, "the catalog names "+source)
// 5. The replicas, replaced one at a time.
if err := s.replace(ctx, j, name); err != nil {
s.fail(j, "replicas", fmt.Errorf("%w. `llmgw ctl model rollback %s` goes back to %s", err, name,
catalog.ShortRevision(o.Revision)))
return
}
s.mu.Lock()
s.updates[name] = UpdateInfo{Model: name, Repo: repo.ID, Current: repo.Revision, Latest: repo.Revision,
Previous: o.Revision, CheckedAt: time.Now()}
s.mu.Unlock()
s.finish(j)
}
// replace restarts a model's replicas onto the catalog in force and waits
// for the new ones, then asks one to answer.
func (s *Server) replace(ctx context.Context, j *Job, name string) error {
var old []string
running := 0
for _, r := range s.deps.Fleet.Replicas() {
if r.Model != name {
continue
}
running++
if r.State == reconciler.Ready {
old = append(old, r.ID)
}
}
if running == 0 {
s.step(j, "replicas", Skipped, "none running; the next to start loads the new weights")
return nil
}
if len(old) == 0 {
s.step(j, "replicas", Skipped, "none ready; those starting load the new weights when they next start")
return nil
}
if err := s.deps.Restart(old...); err != nil {
return err
}
s.step(j, "replicas", Running, fmt.Sprintf("replacing %d, one at a time", len(old)))
deadline := time.Now().Add(s.deps.ReadyTimeout)
for {
left, ready := 0, []reconciler.Replica{}
for _, r := range s.deps.Fleet.Replicas() {
if r.Model != name {
continue
}
if slices.Contains(old, r.ID) {
left++
} else if r.State == reconciler.Ready {
ready = append(ready, r)
}
}
for _, h := range s.deps.Fleet.ModelHealth() {
if h.Model == name && h.Parked {
return fmt.Errorf("the new weights failed to start %d times: %s", h.Failures, h.Reason)
}
}
if left == 0 && len(ready) > 0 {
s.step(j, "replicas", Done, fmt.Sprintf("%d replaced", len(old)))
s.step(j, "smoke test", Running, "asking "+ready[0].ID+" to say hello")
reply, tps, err := s.smoke(ctx, ready[0], name)
if err != nil {
s.step(j, "smoke test", Failed, err.Error())
return err
}
s.mu.Lock()
j.Reply, j.TokensPerSecond = reply, tps
s.mu.Unlock()
s.step(j, "smoke test", Done, fmt.Sprintf("answered at %.1f tokens/s: %q", tps, reply))
return nil
}
s.step(j, "replicas", Running, fmt.Sprintf("%d of %d replaced, %d new ready", len(old)-left, len(old), len(ready)))
if time.Now().After(deadline) {
return fmt.Errorf("the replicas were not replaced after %s", s.deps.ReadyTimeout)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(s.deps.Poll):
}
}
}
// Rollback moves a model back to what its last update replaced, if those
// files are still on the mirror, and restarts its replicas onto them. The
// update is kept as the one to go forward to again.
func (s *Server) Rollback(ctx context.Context, name, by string) (string, error) {
if s.deps.Models == nil || s.deps.Restart == nil {
return "", errors.New("going back needs the store, which this control plane lacks")
}
m, err := s.current(name)
if err != nil {
return "", err
}
if m.Origin == nil || m.Origin.Previous == nil {
return "", fmt.Errorf("%s has not been updated, so there is nothing to go back to", name)
}
prev := m.Origin.Previous
list, err := s.deps.Mirror.List(ctx)
if err != nil {
return "", fmt.Errorf("the mirror: %w", err)
}
if _, ok := placeOf(prev.Source, list); !ok {
return "", fmt.Errorf("the files of %s are no longer on the mirror", catalog.ShortRevision(prev.Revision))
}
back := &catalog.Origin{Repo: m.Origin.Repo, Revision: prev.Revision, Files: prev.Files,
Previous: &catalog.PreviousOrigin{Source: m.Source, Revision: m.Origin.Revision, Files: m.Origin.Files}}
if _, err := s.deps.Models.SetSource(ctx, name, prev.Source, back, by); err != nil {
return "", err
}
var ids []string
for _, r := range s.deps.Fleet.Replicas() {
if r.Model == name && r.State == reconciler.Ready {
ids = append(ids, r.ID)
}
}
msg := fmt.Sprintf("%s runs %s again", name, catalog.ShortRevision(prev.Revision))
if len(ids) > 0 {
if err := s.deps.Restart(ids...); err != nil {
return "", err
}
msg += fmt.Sprintf("; %d replica(s) are being replaced", len(ids))
}
s.mu.Lock()
if u, ok := s.updates[name]; ok {
u.Current, u.Previous = prev.Revision, m.Origin.Revision
u.Available = u.Latest != "" && u.Latest != prev.Revision
s.updates[name] = u
}
s.mu.Unlock()
return msg, nil
}
// UpdateCollector publishes llmgw_model_update_available, 1 for each model
// whose repository has new weights. Registered when the metrics are, before
// there is a server to read: Set gives it one.
type UpdateCollector struct{ s atomic.Pointer[Server] }
var updateDesc = prometheus.NewDesc("llmgw_model_update_available",
"1 when the registry repository a model was served from has published different weights for its files.",
[]string{"model", "repo"}, nil)
// Set gives the collector the server whose checks it reports.
func (c *UpdateCollector) Set(s *Server) { c.s.Store(s) }
// Describe implements prometheus.Collector.
func (c *UpdateCollector) Describe(ch chan<- *prometheus.Desc) { ch <- updateDesc }
// Collect implements prometheus.Collector.
func (c *UpdateCollector) Collect(ch chan<- prometheus.Metric) {
s := c.s.Load()
if s == nil {
return
}
for _, u := range s.Updates() {
v := 0.0
if u.Available {
v = 1
}
ch <- prometheus.MustNewConstMetric(updateDesc, prometheus.GaugeValue, v, u.Model, u.Repo)
}
}