Track the registry's new weights for served models, and move a model to them or back
A model served from the registry records its origin: repository, commit and each file's checksum; one served before is worked out from the mirror's record of where it fetched the file. Every mirror.update_check (6h) the control plane asks whether those files changed there. ctl model updates lists it, ctl model update fetches beside the old files, checks, swaps and replaces replicas one at a time, ctl model rollback goes back. The console shows it on Models and offers the same. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a96b1a54b2
commit
44425193bd
15 files changed
+1393
-33
No files matched your search
@@ -71,6 +71,12 @@ type Deps struct {
|
||||
// ReadyTimeout bounds the wait for a replica: the node fetching the
|
||||
// weights from the mirror and the engine loading them.
|
||||
ReadyTimeout time.Duration
|
||||
|
||||
// For updates: the catalog in force, the service that changes it, and
|
||||
// what replaces replicas. Without them updates are not offered.
|
||||
Current func() []catalog.Model
|
||||
Models Models
|
||||
Restart func(ids ...string) error
|
||||
}
|
||||
|
||||
// Request is what to serve.
|
||||
@@ -116,7 +122,10 @@ type Step struct {
|
||||
|
||||
// Job is one model being served, and where it stands.
|
||||
type Job struct {
|
||||
ID string `json:"id"`
|
||||
ID string `json:"id"`
|
||||
// Kind is "update" for a model moved to new weights, empty for one
|
||||
// served.
|
||||
Kind string `json:"kind,omitempty"`
|
||||
Request Request `json:"request"`
|
||||
State string `json:"state"`
|
||||
Error string `json:"error,omitempty"`
|
||||
@@ -135,6 +144,8 @@ type Server struct {
|
||||
mu sync.Mutex
|
||||
jobs map[string]*Job
|
||||
seq int
|
||||
// updates is what the last checks of the registry found, by model.
|
||||
updates map[string]UpdateInfo
|
||||
}
|
||||
|
||||
// New builds a server.
|
||||
@@ -151,7 +162,7 @@ func New(d Deps) *Server {
|
||||
if d.Log == nil {
|
||||
d.Log = slog.Default()
|
||||
}
|
||||
return &Server{deps: d, jobs: map[string]*Job{}}
|
||||
return &Server{deps: d, jobs: map[string]*Job{}, updates: map[string]UpdateInfo{}}
|
||||
}
|
||||
|
||||
var repoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
@@ -297,6 +308,7 @@ func (s *Server) run(j *Job) {
|
||||
Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores,
|
||||
}},
|
||||
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
|
||||
Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)},
|
||||
}
|
||||
s.step(j, "catalog", Running, "adding "+name)
|
||||
if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil {
|
||||
@@ -388,8 +400,26 @@ func (s *Server) mirror(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSe
|
||||
version := repo.ShortRevision()
|
||||
s.step(j, "mirror", Running, "asking the mirror for "+set.Name)
|
||||
var base string
|
||||
// Files already there from this commit, checksums matching, are not
|
||||
// fetched again: a model moved back and then forward again.
|
||||
have := map[string]bool{}
|
||||
if got, err := s.deps.Mirror.List(ctx); err == nil {
|
||||
for _, st := range got.Models {
|
||||
if st.Model == name && st.Version == version && st.State == mirror.Present {
|
||||
for _, f := range set.Files {
|
||||
if mirror.LastPathElement(f.Path) == st.File && (f.SHA256 == "" || strings.EqualFold(f.SHA256, st.SHA256)) {
|
||||
have[st.File] = true
|
||||
base = mirror.Entry{Model: name, Version: version, File: st.File}.URL(got.Base)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, f := range set.Files {
|
||||
file := mirror.LastPathElement(f.Path)
|
||||
if have[file] {
|
||||
continue
|
||||
}
|
||||
resp, err := s.deps.Mirror.Fetch(ctx, mirror.FetchRequest{
|
||||
Source: repo.DownloadURL("", f.Path), Model: name, Version: version, File: file,
|
||||
})
|
||||
|
||||
@@ -0,0 +1,569 @@
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
// Updates: a repository publishes new weights under the same name — a fixed
|
||||
// chat template, a better quantisation — and nothing told anyone. A model
|
||||
// served from the registry now remembers where it came from; this asks the
|
||||
// registry, every so often, whether the files it runs have changed there,
|
||||
// and moves a model to them on request: fetched into the mirror beside the
|
||||
// old ones, checked, swapped in, its replicas replaced one at a time, and
|
||||
// the old files kept so it can go back.
|
||||
|
||||
// Models is the catalog as an update changes it.
|
||||
type Models interface {
|
||||
Declared(name string) bool
|
||||
SetSource(ctx context.Context, name, source string, origin *catalog.Origin, by string) (catalog.Model, error)
|
||||
SetOrigin(ctx context.Context, name string, origin *catalog.Origin) error
|
||||
}
|
||||
|
||||
// UpdateInfo is what the registry has for a model against what it runs.
|
||||
type UpdateInfo struct {
|
||||
Model string `json:"model"`
|
||||
Repo string `json:"repo,omitempty"`
|
||||
// Current is the commit the model runs, Latest the repository's now.
|
||||
Current string `json:"current,omitempty"`
|
||||
Latest string `json:"latest,omitempty"`
|
||||
// Changed are the model's files whose weights differ at Latest, and
|
||||
// Missing those Latest no longer has at all.
|
||||
Changed []string `json:"changed,omitempty"`
|
||||
Missing []string `json:"missing,omitempty"`
|
||||
// Bytes is what an update downloads.
|
||||
Bytes int64 `json:"bytes,omitempty"`
|
||||
// Available is true when the model can be moved to Latest.
|
||||
Available bool `json:"available"`
|
||||
// Previous is the commit an update replaced, which it can go back to.
|
||||
Previous string `json:"previous,omitempty"`
|
||||
CheckedAt time.Time `json:"checked_at"`
|
||||
Err string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// place is where a model's weights sit on the mirror.
|
||||
type place struct {
|
||||
model, version string
|
||||
files []mirror.Status
|
||||
}
|
||||
|
||||
// placeOf finds a source on the mirror: a file, or a directory of shards.
|
||||
func placeOf(source string, list mirror.ModelsResponse) (place, bool) {
|
||||
base := strings.TrimSuffix(list.Base, "/") + "/"
|
||||
if list.Base == "" || !strings.HasPrefix(source, base) {
|
||||
return place{}, false
|
||||
}
|
||||
parts := strings.SplitN(strings.TrimPrefix(source, base), "/", 3)
|
||||
if len(parts) < 2 {
|
||||
return place{}, false
|
||||
}
|
||||
p := place{model: parts[0], version: parts[1]}
|
||||
file := ""
|
||||
if len(parts) == 3 {
|
||||
file = parts[2]
|
||||
}
|
||||
for _, st := range list.Models {
|
||||
if st.Model == p.model && st.Version == p.version && (file == "" || st.File == file) {
|
||||
p.files = append(p.files, st)
|
||||
}
|
||||
}
|
||||
return p, len(p.files) > 0
|
||||
}
|
||||
|
||||
// parseResolve reads a registry download URL: base/owner/name/resolve/rev/path.
|
||||
func parseResolve(u string) (repo, rev, path string, ok bool) {
|
||||
pu, err := url.Parse(u)
|
||||
if err != nil {
|
||||
return "", "", "", false
|
||||
}
|
||||
i := strings.Index(pu.Path, "/resolve/")
|
||||
if i < 0 {
|
||||
return "", "", "", false
|
||||
}
|
||||
repo = strings.Trim(pu.Path[:i], "/")
|
||||
if strings.Count(repo, "/") != 1 {
|
||||
return "", "", "", false
|
||||
}
|
||||
rev, path, ok = strings.Cut(pu.Path[i+len("/resolve/"):], "/")
|
||||
return repo, rev, path, ok && rev != "" && path != ""
|
||||
}
|
||||
|
||||
// inferOrigin works a model's origin out from what the mirror recorded of
|
||||
// where it fetched each file, for a model served before origins were kept.
|
||||
func inferOrigin(p place) (*catalog.Origin, bool) {
|
||||
var o catalog.Origin
|
||||
for _, st := range p.files {
|
||||
repo, rev, path, ok := parseResolve(st.Source)
|
||||
if !ok || (o.Repo != "" && (repo != o.Repo || rev != o.Revision)) {
|
||||
return nil, false
|
||||
}
|
||||
o.Repo, o.Revision = repo, rev
|
||||
o.Files = append(o.Files, catalog.OriginFile{Path: path, SHA256: st.SHA256, Bytes: st.BytesDone})
|
||||
}
|
||||
return &o, o.Repo != ""
|
||||
}
|
||||
|
||||
func originFiles(set hf.FileSet) []catalog.OriginFile {
|
||||
out := make([]catalog.OriginFile, 0, len(set.Files))
|
||||
for _, f := range set.Files {
|
||||
out = append(out, catalog.OriginFile{Path: f.Path, SHA256: f.SHA256, Bytes: f.Bytes})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// originOf is a model's origin: recorded, or worked out from the mirror and
|
||||
// then recorded.
|
||||
func (s *Server) originOf(ctx context.Context, m catalog.Model, list mirror.ModelsResponse) *catalog.Origin {
|
||||
if m.Origin != nil && m.Origin.Repo != "" {
|
||||
return m.Origin
|
||||
}
|
||||
p, ok := placeOf(m.Source, list)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
o, ok := inferOrigin(p)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if s.deps.Models != nil {
|
||||
if err := s.deps.Models.SetOrigin(ctx, m.Name, o); err != nil {
|
||||
s.deps.Log.Debug("could not record a model's origin", "model", m.Name, "err", err)
|
||||
}
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// compare says what the registry's repository has against an origin.
|
||||
func compare(m catalog.Model, o *catalog.Origin, repo hf.Repo) UpdateInfo {
|
||||
u := UpdateInfo{Model: m.Name, Repo: o.Repo, Current: o.Revision, Latest: repo.Revision, CheckedAt: time.Now()}
|
||||
if o.Previous != nil {
|
||||
u.Previous = o.Previous.Revision
|
||||
}
|
||||
if repo.Revision == o.Revision {
|
||||
return u
|
||||
}
|
||||
for _, f := range o.Files {
|
||||
i := slices.IndexFunc(repo.Files, func(g hf.File) bool { return g.Path == f.Path })
|
||||
if i < 0 {
|
||||
u.Missing = append(u.Missing, f.Path)
|
||||
continue
|
||||
}
|
||||
g := repo.Files[i]
|
||||
if (f.SHA256 != "" && g.SHA256 != "" && !strings.EqualFold(f.SHA256, g.SHA256)) ||
|
||||
(f.SHA256 == "" && f.Bytes != 0 && f.Bytes != g.Bytes) {
|
||||
u.Changed = append(u.Changed, f.Path)
|
||||
u.Bytes += g.Bytes
|
||||
}
|
||||
}
|
||||
u.Available = len(u.Changed) > 0 && len(u.Missing) == 0
|
||||
return u
|
||||
}
|
||||
|
||||
// CheckUpdates asks the registry about every model with an origin.
|
||||
func (s *Server) CheckUpdates(ctx context.Context) {
|
||||
if s.deps.Current == nil {
|
||||
return
|
||||
}
|
||||
list, err := s.deps.Mirror.List(ctx)
|
||||
if err != nil {
|
||||
s.deps.Log.Warn("checking for model updates: the mirror", "err", err)
|
||||
return
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, m := range s.deps.Current() {
|
||||
o := s.originOf(ctx, m, list)
|
||||
if o == nil {
|
||||
continue
|
||||
}
|
||||
seen[m.Name] = true
|
||||
s.checkOne(ctx, m, o)
|
||||
}
|
||||
s.mu.Lock()
|
||||
for name := range s.updates {
|
||||
if !seen[name] {
|
||||
delete(s.updates, name)
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *Server) checkOne(ctx context.Context, m catalog.Model, o *catalog.Origin) UpdateInfo {
|
||||
repo, err := s.deps.Registry.Repo(ctx, o.Repo)
|
||||
u := UpdateInfo{Model: m.Name, Repo: o.Repo, Current: o.Revision, CheckedAt: time.Now()}
|
||||
if err != nil {
|
||||
u.Err = err.Error()
|
||||
} else {
|
||||
u = compare(m, o, repo)
|
||||
if u.Available {
|
||||
s.deps.Log.Info("a model's repository has new weights", "model", m.Name, "repo", o.Repo,
|
||||
"current", catalog.ShortRevision(o.Revision), "latest", catalog.ShortRevision(repo.Revision))
|
||||
}
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.updates[m.Name] = u
|
||||
s.mu.Unlock()
|
||||
return u
|
||||
}
|
||||
|
||||
// CheckModel asks the registry about one model now.
|
||||
func (s *Server) CheckModel(ctx context.Context, name string) (UpdateInfo, error) {
|
||||
m, err := s.current(name)
|
||||
if err != nil {
|
||||
return UpdateInfo{}, err
|
||||
}
|
||||
list, err := s.deps.Mirror.List(ctx)
|
||||
if err != nil {
|
||||
return UpdateInfo{}, fmt.Errorf("the mirror: %w", err)
|
||||
}
|
||||
o := s.originOf(ctx, m, list)
|
||||
if o == nil {
|
||||
return UpdateInfo{}, fmt.Errorf("%s did not come from the registry through the mirror, so there is nothing to compare", name)
|
||||
}
|
||||
return s.checkOne(ctx, m, o), nil
|
||||
}
|
||||
|
||||
// Updates is what the last checks found, by model.
|
||||
func (s *Server) Updates() map[string]UpdateInfo {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := make(map[string]UpdateInfo, len(s.updates))
|
||||
for k, v := range s.updates {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// WatchUpdates checks every so often until ctx ends; zero never checks.
|
||||
func (s *Server) WatchUpdates(ctx context.Context, every time.Duration) {
|
||||
if every <= 0 {
|
||||
return
|
||||
}
|
||||
// A little after starting, when the catalog is in force and the mirror
|
||||
// answering, rather than in the middle of starting.
|
||||
wait := 2 * time.Minute
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(wait):
|
||||
}
|
||||
s.CheckUpdates(ctx)
|
||||
wait = every
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) current(name string) (catalog.Model, error) {
|
||||
if s.deps.Current == nil {
|
||||
return catalog.Model{}, errors.New("this control plane cannot read its catalog here")
|
||||
}
|
||||
for _, m := range s.deps.Current() {
|
||||
if m.Name == name {
|
||||
return m, nil
|
||||
}
|
||||
}
|
||||
return catalog.Model{}, fmt.Errorf("no model called %s", name)
|
||||
}
|
||||
|
||||
// StartUpdate moves a model to its repository's latest weights, as a job.
|
||||
func (s *Server) StartUpdate(name, by string) (Job, error) {
|
||||
if s.deps.Models == nil || s.deps.Restart == nil {
|
||||
return Job{}, errors.New("updating needs the store, which this control plane lacks")
|
||||
}
|
||||
if _, err := s.current(name); err != nil {
|
||||
return Job{}, err
|
||||
}
|
||||
if s.deps.Models.Declared(name) {
|
||||
return Job{}, fmt.Errorf("%s is declared in server.yaml: serve the new weights there, by its source, "+
|
||||
"or they would revert on the next restart", name)
|
||||
}
|
||||
s.mu.Lock()
|
||||
for _, j := range s.jobs {
|
||||
if j.Kind == "update" && j.Model == name && j.State == Running {
|
||||
s.mu.Unlock()
|
||||
return Job{}, fmt.Errorf("%s is already being updated: job %s", name, j.ID)
|
||||
}
|
||||
}
|
||||
s.seq++
|
||||
j := &Job{ID: fmt.Sprintf("update-%d-%d", time.Now().Unix(), s.seq), Kind: "update", Model: name,
|
||||
Request: Request{Name: name, By: by}, State: Running, Started: time.Now()}
|
||||
s.jobs[j.ID] = j
|
||||
s.mu.Unlock()
|
||||
go s.runUpdate(j)
|
||||
return s.snapshot(j), nil
|
||||
}
|
||||
|
||||
func (s *Server) runUpdate(j *Job) {
|
||||
ctx := context.Background()
|
||||
name := j.Model
|
||||
|
||||
// 1. The registry: which files, at which commit.
|
||||
s.step(j, "registry", Running, "reading where "+name+" came from")
|
||||
m, err := s.current(name)
|
||||
if err != nil {
|
||||
s.fail(j, "registry", err)
|
||||
return
|
||||
}
|
||||
list, err := s.deps.Mirror.List(ctx)
|
||||
if err != nil {
|
||||
s.fail(j, "registry", fmt.Errorf("the mirror: %w", err))
|
||||
return
|
||||
}
|
||||
o := s.originOf(ctx, m, list)
|
||||
p, onMirror := placeOf(m.Source, list)
|
||||
if o == nil || !onMirror {
|
||||
s.fail(j, "registry", fmt.Errorf("%s did not come from the registry through the mirror", name))
|
||||
return
|
||||
}
|
||||
repo, err := s.deps.Registry.Repo(ctx, o.Repo)
|
||||
if err != nil {
|
||||
s.fail(j, "registry", err)
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
j.Request.Repo = repo.ID
|
||||
s.mu.Unlock()
|
||||
u := compare(m, o, repo)
|
||||
switch {
|
||||
case repo.Revision == o.Revision:
|
||||
s.fail(j, "registry", fmt.Errorf("%s already runs %s's latest commit, %s", name, repo.ID, catalog.ShortRevision(repo.Revision)))
|
||||
return
|
||||
case len(u.Missing) > 0:
|
||||
s.fail(j, "registry", fmt.Errorf("%s at %s no longer has %s; choose one of its files and serve it under a new name",
|
||||
repo.ID, repo.ShortRevision(), strings.Join(u.Missing, ", ")))
|
||||
return
|
||||
case len(u.Changed) == 0:
|
||||
s.fail(j, "registry", fmt.Errorf("%s has a newer commit, %s, but the files %s runs are the same in it",
|
||||
repo.ID, repo.ShortRevision(), name))
|
||||
return
|
||||
case repo.Gated.Is():
|
||||
s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", repo.ID))
|
||||
return
|
||||
}
|
||||
set := hf.FileSet{}
|
||||
for _, f := range o.Files {
|
||||
g := repo.Files[slices.IndexFunc(repo.Files, func(g hf.File) bool { return g.Path == f.Path })]
|
||||
set.Files = append(set.Files, g)
|
||||
set.Bytes += g.Bytes
|
||||
}
|
||||
set.Name, set.Shards = set.Files[0].Path, len(set.Files)
|
||||
s.step(j, "registry", Done, fmt.Sprintf("%s: %s → %s, %d of %d file(s) changed, %.1f GB",
|
||||
repo.ID, catalog.ShortRevision(o.Revision), repo.ShortRevision(), len(u.Changed), len(o.Files), gb(set.Bytes)))
|
||||
|
||||
// 2. The mirror, beside the files in use.
|
||||
source, err := s.mirror(ctx, j, repo, set, p.model)
|
||||
if err != nil {
|
||||
s.fail(j, "mirror", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Whether the nodes load it, and whether it still fits what is
|
||||
// reserved for it.
|
||||
s.step(j, "check", Running, "reading its header from the mirror")
|
||||
detail := "a set of shards; not read"
|
||||
if !strings.HasSuffix(source, "/") {
|
||||
v := gguf.Check(ctx, source, s.deps.LlamaVersion)
|
||||
if !v.Servable {
|
||||
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason))
|
||||
return
|
||||
}
|
||||
detail = orUnknown(v.Header.Architecture)
|
||||
if len(m.Profiles) > 0 {
|
||||
ctxPool := 16384
|
||||
for i, a := range m.Profiles[0].Args {
|
||||
if a == "--ctx-size" && i+1 < len(m.Profiles[0].Args) {
|
||||
fmt.Sscan(m.Profiles[0].Args[i+1], &ctxPool)
|
||||
}
|
||||
}
|
||||
need := int((v.Header.MemoryBytes(set.Bytes, ctxPool) + 1<<30 - 1) >> 30)
|
||||
if have := m.Profiles[0].MinMemoryGB; need > have && have > 0 {
|
||||
detail += fmt.Sprintf("; needs about %d GB and %d is reserved: raise it if replicas fail", need, have)
|
||||
}
|
||||
}
|
||||
}
|
||||
s.step(j, "check", Done, detail)
|
||||
|
||||
// 4. The catalog: the new source, and the old one kept to go back to.
|
||||
s.step(j, "catalog", Running, "moving "+name+" to "+repo.ShortRevision())
|
||||
next := &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set),
|
||||
Previous: &catalog.PreviousOrigin{Source: m.Source, Revision: o.Revision, Files: o.Files}}
|
||||
if _, err := s.deps.Models.SetSource(ctx, name, source, next, j.Request.By); err != nil {
|
||||
s.fail(j, "catalog", err)
|
||||
return
|
||||
}
|
||||
s.step(j, "catalog", Done, "the catalog names "+source)
|
||||
|
||||
// 5. The replicas, replaced one at a time.
|
||||
if err := s.replace(ctx, j, name); err != nil {
|
||||
s.fail(j, "replicas", fmt.Errorf("%w. `llmgw ctl model rollback %s` goes back to %s", err, name,
|
||||
catalog.ShortRevision(o.Revision)))
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.updates[name] = UpdateInfo{Model: name, Repo: repo.ID, Current: repo.Revision, Latest: repo.Revision,
|
||||
Previous: o.Revision, CheckedAt: time.Now()}
|
||||
s.mu.Unlock()
|
||||
s.finish(j)
|
||||
}
|
||||
|
||||
// replace restarts a model's replicas onto the catalog in force and waits
|
||||
// for the new ones, then asks one to answer.
|
||||
func (s *Server) replace(ctx context.Context, j *Job, name string) error {
|
||||
var old []string
|
||||
running := 0
|
||||
for _, r := range s.deps.Fleet.Replicas() {
|
||||
if r.Model != name {
|
||||
continue
|
||||
}
|
||||
running++
|
||||
if r.State == reconciler.Ready {
|
||||
old = append(old, r.ID)
|
||||
}
|
||||
}
|
||||
if running == 0 {
|
||||
s.step(j, "replicas", Skipped, "none running; the next to start loads the new weights")
|
||||
return nil
|
||||
}
|
||||
if len(old) == 0 {
|
||||
s.step(j, "replicas", Skipped, "none ready; those starting load the new weights when they next start")
|
||||
return nil
|
||||
}
|
||||
if err := s.deps.Restart(old...); err != nil {
|
||||
return err
|
||||
}
|
||||
s.step(j, "replicas", Running, fmt.Sprintf("replacing %d, one at a time", len(old)))
|
||||
deadline := time.Now().Add(s.deps.ReadyTimeout)
|
||||
for {
|
||||
left, ready := 0, []reconciler.Replica{}
|
||||
for _, r := range s.deps.Fleet.Replicas() {
|
||||
if r.Model != name {
|
||||
continue
|
||||
}
|
||||
if slices.Contains(old, r.ID) {
|
||||
left++
|
||||
} else if r.State == reconciler.Ready {
|
||||
ready = append(ready, r)
|
||||
}
|
||||
}
|
||||
for _, h := range s.deps.Fleet.ModelHealth() {
|
||||
if h.Model == name && h.Parked {
|
||||
return fmt.Errorf("the new weights failed to start %d times: %s", h.Failures, h.Reason)
|
||||
}
|
||||
}
|
||||
if left == 0 && len(ready) > 0 {
|
||||
s.step(j, "replicas", Done, fmt.Sprintf("%d replaced", len(old)))
|
||||
s.step(j, "smoke test", Running, "asking "+ready[0].ID+" to say hello")
|
||||
reply, tps, err := s.smoke(ctx, ready[0], name)
|
||||
if err != nil {
|
||||
s.step(j, "smoke test", Failed, err.Error())
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
j.Reply, j.TokensPerSecond = reply, tps
|
||||
s.mu.Unlock()
|
||||
s.step(j, "smoke test", Done, fmt.Sprintf("answered at %.1f tokens/s: %q", tps, reply))
|
||||
return nil
|
||||
}
|
||||
s.step(j, "replicas", Running, fmt.Sprintf("%d of %d replaced, %d new ready", len(old)-left, len(old), len(ready)))
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("the replicas were not replaced after %s", s.deps.ReadyTimeout)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(s.deps.Poll):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Rollback moves a model back to what its last update replaced, if those
|
||||
// files are still on the mirror, and restarts its replicas onto them. The
|
||||
// update is kept as the one to go forward to again.
|
||||
func (s *Server) Rollback(ctx context.Context, name, by string) (string, error) {
|
||||
if s.deps.Models == nil || s.deps.Restart == nil {
|
||||
return "", errors.New("going back needs the store, which this control plane lacks")
|
||||
}
|
||||
m, err := s.current(name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if m.Origin == nil || m.Origin.Previous == nil {
|
||||
return "", fmt.Errorf("%s has not been updated, so there is nothing to go back to", name)
|
||||
}
|
||||
prev := m.Origin.Previous
|
||||
list, err := s.deps.Mirror.List(ctx)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the mirror: %w", err)
|
||||
}
|
||||
if _, ok := placeOf(prev.Source, list); !ok {
|
||||
return "", fmt.Errorf("the files of %s are no longer on the mirror", catalog.ShortRevision(prev.Revision))
|
||||
}
|
||||
back := &catalog.Origin{Repo: m.Origin.Repo, Revision: prev.Revision, Files: prev.Files,
|
||||
Previous: &catalog.PreviousOrigin{Source: m.Source, Revision: m.Origin.Revision, Files: m.Origin.Files}}
|
||||
if _, err := s.deps.Models.SetSource(ctx, name, prev.Source, back, by); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var ids []string
|
||||
for _, r := range s.deps.Fleet.Replicas() {
|
||||
if r.Model == name && r.State == reconciler.Ready {
|
||||
ids = append(ids, r.ID)
|
||||
}
|
||||
}
|
||||
msg := fmt.Sprintf("%s runs %s again", name, catalog.ShortRevision(prev.Revision))
|
||||
if len(ids) > 0 {
|
||||
if err := s.deps.Restart(ids...); err != nil {
|
||||
return "", err
|
||||
}
|
||||
msg += fmt.Sprintf("; %d replica(s) are being replaced", len(ids))
|
||||
}
|
||||
s.mu.Lock()
|
||||
if u, ok := s.updates[name]; ok {
|
||||
u.Current, u.Previous = prev.Revision, m.Origin.Revision
|
||||
u.Available = u.Latest != "" && u.Latest != prev.Revision
|
||||
s.updates[name] = u
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
// UpdateCollector publishes llmgw_model_update_available, 1 for each model
|
||||
// whose repository has new weights. Registered when the metrics are, before
|
||||
// there is a server to read: Set gives it one.
|
||||
type UpdateCollector struct{ s atomic.Pointer[Server] }
|
||||
|
||||
var updateDesc = prometheus.NewDesc("llmgw_model_update_available",
|
||||
"1 when the registry repository a model was served from has published different weights for its files.",
|
||||
[]string{"model", "repo"}, nil)
|
||||
|
||||
// Set gives the collector the server whose checks it reports.
|
||||
func (c *UpdateCollector) Set(s *Server) { c.s.Store(s) }
|
||||
|
||||
// Describe implements prometheus.Collector.
|
||||
func (c *UpdateCollector) Describe(ch chan<- *prometheus.Desc) { ch <- updateDesc }
|
||||
|
||||
// Collect implements prometheus.Collector.
|
||||
func (c *UpdateCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
s := c.s.Load()
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
for _, u := range s.Updates() {
|
||||
v := 0.0
|
||||
if u.Available {
|
||||
v = 1
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(updateDesc, prometheus.GaugeValue, v, u.Model, u.Repo)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
)
|
||||
|
||||
const (
|
||||
oldRev = "abc123456789aaaa"
|
||||
newRev = "def987654321bbbb"
|
||||
)
|
||||
|
||||
type movingRegistry struct {
|
||||
mu sync.Mutex
|
||||
repo hf.Repo
|
||||
}
|
||||
|
||||
func (r *movingRegistry) Repo(context.Context, string) (hf.Repo, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.repo, nil
|
||||
}
|
||||
|
||||
// updMirror is a mirror that records where it fetched each file from.
|
||||
type updMirror struct {
|
||||
mu sync.Mutex
|
||||
base string
|
||||
held []mirror.Status
|
||||
sums map[string]string
|
||||
}
|
||||
|
||||
func (m *updMirror) Fetch(_ context.Context, r mirror.FetchRequest) (mirror.FetchResponse, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.held = append(m.held, mirror.Status{Model: r.Model, Version: r.Version, File: r.File, Source: r.Source,
|
||||
State: mirror.Present, BytesDone: 1 << 30, SHA256: m.sums[r.Version+"/"+r.File]})
|
||||
return mirror.FetchResponse{URL: mirror.Entry{Model: r.Model, Version: r.Version, File: r.File}.URL(m.base)}, nil
|
||||
}
|
||||
|
||||
func (m *updMirror) List(context.Context) (mirror.ModelsResponse, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return mirror.ModelsResponse{Base: m.base, Models: append([]mirror.Status{}, m.held...)}, nil
|
||||
}
|
||||
|
||||
type updModels struct {
|
||||
mu sync.Mutex
|
||||
models map[string]catalog.Model
|
||||
declared map[string]bool
|
||||
}
|
||||
|
||||
func (c *updModels) Declared(n string) bool { return c.declared[n] }
|
||||
func (c *updModels) SetSource(_ context.Context, n, src string, o *catalog.Origin, _ string) (catalog.Model, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
m := c.models[n]
|
||||
m.Source, m.Origin = src, o
|
||||
c.models[n] = m
|
||||
return m, nil
|
||||
}
|
||||
func (c *updModels) SetOrigin(_ context.Context, n string, o *catalog.Origin) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
m := c.models[n]
|
||||
m.Origin = o
|
||||
c.models[n] = m
|
||||
return nil
|
||||
}
|
||||
func (c *updModels) all() []catalog.Model {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
var out []catalog.Model
|
||||
for _, m := range c.models {
|
||||
out = append(out, m)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// restartFleet replaces a restarted replica with a new one, ready.
|
||||
type restartFleet struct {
|
||||
mu sync.Mutex
|
||||
reps []reconciler.Replica
|
||||
engine string
|
||||
seq int
|
||||
}
|
||||
|
||||
func (f *restartFleet) Replicas() []reconciler.Replica {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]reconciler.Replica{}, f.reps...)
|
||||
}
|
||||
func (f *restartFleet) ModelHealth() []reconciler.ModelHealth { return nil }
|
||||
func (f *restartFleet) EngineKey(string) string { return "" }
|
||||
func (f *restartFleet) Restart(ids ...string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
for _, id := range ids {
|
||||
for i, r := range f.reps {
|
||||
if r.ID == id {
|
||||
f.seq++
|
||||
f.reps[i] = reconciler.Replica{ID: r.Model + "-new" + string(rune('0'+f.seq)), Model: r.Model,
|
||||
State: reconciler.Ready, Endpoint: f.engine}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func updSetup(t *testing.T) (*Server, *updMirror, *updModels, *movingRegistry, *restartFleet) {
|
||||
t.Helper()
|
||||
header := ggufHeader()
|
||||
files := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(header) }))
|
||||
t.Cleanup(files.Close)
|
||||
engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"Hi"}}],"usage":{"completion_tokens":1}}`))
|
||||
}))
|
||||
t.Cleanup(engine.Close)
|
||||
|
||||
mir := &updMirror{base: files.URL, sums: map[string]string{
|
||||
"abc12345/q4.gguf": "aaaa", "def98765/q4.gguf": "bbbb"}}
|
||||
// What was fetched before origins were kept: the mirror alone says where from.
|
||||
_, _ = mir.Fetch(context.Background(), mirror.FetchRequest{Model: "qwen", Version: "abc12345", File: "q4.gguf",
|
||||
Source: "https://huggingface.co/o/r/resolve/" + oldRev + "/q4.gguf"})
|
||||
models := &updModels{models: map[string]catalog.Model{"qwen": {Name: "qwen",
|
||||
Source: files.URL + "/qwen/abc12345/q4.gguf",
|
||||
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 8}}}}, declared: map[string]bool{}}
|
||||
reg := &movingRegistry{repo: hf.Repo{ID: "o/r", Revision: newRev, Files: []hf.File{
|
||||
{Path: "q4.gguf", SHA256: "bbbb", Bytes: 2 << 30}, {Path: "README.md", Bytes: 10}}}}
|
||||
fleet := &restartFleet{engine: engine.URL, reps: []reconciler.Replica{
|
||||
{ID: "qwen-1", Model: "qwen", State: reconciler.Ready, Endpoint: engine.URL},
|
||||
{ID: "qwen-2", Model: "qwen", State: reconciler.Ready, Endpoint: engine.URL}}}
|
||||
s := New(Deps{Registry: reg, Mirror: mir, Catalog: &fakeCatalog{}, Fleet: fleet,
|
||||
Current: models.all, Models: models, Restart: fleet.Restart,
|
||||
Poll: 10 * time.Millisecond, ReadyTimeout: 5 * time.Second})
|
||||
return s, mir, models, reg, fleet
|
||||
}
|
||||
|
||||
func TestNewWeightsInTheRepositoryAreFoundForAModelServedBeforeOriginsWereKept(t *testing.T) {
|
||||
s, _, models, _, _ := updSetup(t)
|
||||
s.CheckUpdates(context.Background())
|
||||
u := s.Updates()["qwen"]
|
||||
if !u.Available || u.Repo != "o/r" || u.Current != oldRev || u.Latest != newRev ||
|
||||
len(u.Changed) != 1 || u.Changed[0] != "q4.gguf" || u.Bytes != 2<<30 {
|
||||
t.Fatalf("%+v", u)
|
||||
}
|
||||
if o := models.models["qwen"].Origin; o == nil || o.Repo != "o/r" || o.Files[0].SHA256 != "aaaa" {
|
||||
t.Errorf("the origin worked out was not recorded: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANewCommitThatLeavesTheWeightsAloneIsNotAnUpdate(t *testing.T) {
|
||||
s, mir, _, _, _ := updSetup(t)
|
||||
mir.held[0].SHA256 = "bbbb"
|
||||
u, err := s.CheckModel(context.Background(), "qwen")
|
||||
if err != nil || u.Available || u.Latest != newRev || updateSays(u) != "same" {
|
||||
t.Fatalf("%+v %v", u, err)
|
||||
}
|
||||
}
|
||||
|
||||
func updateSays(u UpdateInfo) string {
|
||||
if !u.Available && u.Latest != u.Current && len(u.Missing) == 0 {
|
||||
return "same"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestAnUpdateFetchesBesideTheOldReplacesTheReplicasAndCanGoBack(t *testing.T) {
|
||||
s, mir, models, _, fleet := updSetup(t)
|
||||
oldSource := models.models["qwen"].Source
|
||||
job, err := s.StartUpdate("qwen", "tester")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := wait(t, s, job.ID)
|
||||
if j.State != Done {
|
||||
t.Fatalf("%s: %+v", j.Error, j.Steps)
|
||||
}
|
||||
m := models.models["qwen"]
|
||||
if !strings.HasSuffix(m.Source, "/qwen/def98765/q4.gguf") || m.Origin.Revision != newRev ||
|
||||
m.Origin.Previous == nil || m.Origin.Previous.Source != oldSource {
|
||||
t.Fatalf("catalog: %s %+v", m.Source, m.Origin)
|
||||
}
|
||||
if len(mir.held) != 2 {
|
||||
t.Errorf("the old files did not stay on the mirror beside the new: %+v", mir.held)
|
||||
}
|
||||
for _, r := range fleet.Replicas() {
|
||||
if !strings.Contains(r.ID, "new") {
|
||||
t.Errorf("%s was not replaced", r.ID)
|
||||
}
|
||||
}
|
||||
if u := s.Updates()["qwen"]; u.Available || u.Current != newRev {
|
||||
t.Errorf("still offered: %+v", u)
|
||||
}
|
||||
|
||||
// And back.
|
||||
if _, err := s.Rollback(context.Background(), "qwen", "tester"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m = models.models["qwen"]
|
||||
if m.Source != oldSource || m.Origin.Revision != oldRev || m.Origin.Previous.Revision != newRev {
|
||||
t.Fatalf("rollback: %s %+v", m.Source, m.Origin)
|
||||
}
|
||||
if u := s.Updates()["qwen"]; !u.Available {
|
||||
t.Errorf("the update it went back from is not offered again: %+v", u)
|
||||
}
|
||||
|
||||
// And forward again, from files already on the mirror.
|
||||
job, err = s.StartUpdate("qwen", "tester")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if j := wait(t, s, job.ID); j.State != Done || models.models["qwen"].Origin.Revision != newRev {
|
||||
t.Fatalf("%s: %+v", j.Error, j.Steps)
|
||||
}
|
||||
// Then there is nothing newer: refused, saying why.
|
||||
job, _ = s.StartUpdate("qwen", "tester")
|
||||
if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "already runs") {
|
||||
t.Fatalf("%+v", j)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelFromConfigurationIsNotUpdatedBehindItsBack(t *testing.T) {
|
||||
s, _, models, _, _ := updSetup(t)
|
||||
models.declared["qwen"] = true
|
||||
if _, err := s.StartUpdate("qwen", "t"); err == nil || !strings.Contains(err.Error(), "server.yaml") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUpdateToFilesTheRepositoryDroppedIsRefused(t *testing.T) {
|
||||
s, _, _, reg, _ := updSetup(t)
|
||||
reg.repo.Files = []hf.File{{Path: "q4-v2.gguf", SHA256: "cccc"}}
|
||||
job, _ := s.StartUpdate("qwen", "t")
|
||||
if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "no longer has q4.gguf") {
|
||||
t.Fatalf("%+v", j)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryDownloadURLsAreRead(t *testing.T) {
|
||||
for u, want := range map[string][3]string{
|
||||
"https://huggingface.co/bartowski/Qwen-GGUF/resolve/abc/sub/f.gguf": {"bartowski/Qwen-GGUF", "abc", "sub/f.gguf"},
|
||||
"https://hf.internal/mirror/x/resolve/main/f.gguf": {"mirror/x", "main", "f.gguf"},
|
||||
} {
|
||||
repo, rev, path, ok := parseResolve(u)
|
||||
if !ok || [3]string{repo, rev, path} != want {
|
||||
t.Errorf("%s: %s %s %s %v", u, repo, rev, path, ok)
|
||||
}
|
||||
}
|
||||
for _, u := range []string{"https://example.com/f.gguf", "https://huggingface.co/a/b/c/resolve/x/f", ":"} {
|
||||
if _, _, _, ok := parseResolve(u); ok {
|
||||
t.Errorf("%s was read", u)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user