A model served from the registry records its origin: repository, commit and each file's checksum; one served before is worked out from the mirror's record of where it fetched the file. Every mirror.update_check (6h) the control plane asks whether those files changed there. ctl model updates lists it, ctl model update fetches beside the old files, checks, swaps and replaces replicas one at a time, ctl model rollback goes back. The console shows it on Models and offers the same. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
265 lines
8.7 KiB
Go
265 lines
8.7 KiB
Go
package modelserve
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
|
)
|
|
|
|
const (
|
|
oldRev = "abc123456789aaaa"
|
|
newRev = "def987654321bbbb"
|
|
)
|
|
|
|
type movingRegistry struct {
|
|
mu sync.Mutex
|
|
repo hf.Repo
|
|
}
|
|
|
|
func (r *movingRegistry) Repo(context.Context, string) (hf.Repo, error) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return r.repo, nil
|
|
}
|
|
|
|
// updMirror is a mirror that records where it fetched each file from.
|
|
type updMirror struct {
|
|
mu sync.Mutex
|
|
base string
|
|
held []mirror.Status
|
|
sums map[string]string
|
|
}
|
|
|
|
func (m *updMirror) Fetch(_ context.Context, r mirror.FetchRequest) (mirror.FetchResponse, error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
m.held = append(m.held, mirror.Status{Model: r.Model, Version: r.Version, File: r.File, Source: r.Source,
|
|
State: mirror.Present, BytesDone: 1 << 30, SHA256: m.sums[r.Version+"/"+r.File]})
|
|
return mirror.FetchResponse{URL: mirror.Entry{Model: r.Model, Version: r.Version, File: r.File}.URL(m.base)}, nil
|
|
}
|
|
|
|
func (m *updMirror) List(context.Context) (mirror.ModelsResponse, error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
return mirror.ModelsResponse{Base: m.base, Models: append([]mirror.Status{}, m.held...)}, nil
|
|
}
|
|
|
|
type updModels struct {
|
|
mu sync.Mutex
|
|
models map[string]catalog.Model
|
|
declared map[string]bool
|
|
}
|
|
|
|
func (c *updModels) Declared(n string) bool { return c.declared[n] }
|
|
func (c *updModels) SetSource(_ context.Context, n, src string, o *catalog.Origin, _ string) (catalog.Model, error) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
m := c.models[n]
|
|
m.Source, m.Origin = src, o
|
|
c.models[n] = m
|
|
return m, nil
|
|
}
|
|
func (c *updModels) SetOrigin(_ context.Context, n string, o *catalog.Origin) error {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
m := c.models[n]
|
|
m.Origin = o
|
|
c.models[n] = m
|
|
return nil
|
|
}
|
|
func (c *updModels) all() []catalog.Model {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
var out []catalog.Model
|
|
for _, m := range c.models {
|
|
out = append(out, m)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// restartFleet replaces a restarted replica with a new one, ready.
|
|
type restartFleet struct {
|
|
mu sync.Mutex
|
|
reps []reconciler.Replica
|
|
engine string
|
|
seq int
|
|
}
|
|
|
|
func (f *restartFleet) Replicas() []reconciler.Replica {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return append([]reconciler.Replica{}, f.reps...)
|
|
}
|
|
func (f *restartFleet) ModelHealth() []reconciler.ModelHealth { return nil }
|
|
func (f *restartFleet) EngineKey(string) string { return "" }
|
|
func (f *restartFleet) Restart(ids ...string) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
for _, id := range ids {
|
|
for i, r := range f.reps {
|
|
if r.ID == id {
|
|
f.seq++
|
|
f.reps[i] = reconciler.Replica{ID: r.Model + "-new" + string(rune('0'+f.seq)), Model: r.Model,
|
|
State: reconciler.Ready, Endpoint: f.engine}
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func updSetup(t *testing.T) (*Server, *updMirror, *updModels, *movingRegistry, *restartFleet) {
|
|
t.Helper()
|
|
header := ggufHeader()
|
|
files := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(header) }))
|
|
t.Cleanup(files.Close)
|
|
engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"Hi"}}],"usage":{"completion_tokens":1}}`))
|
|
}))
|
|
t.Cleanup(engine.Close)
|
|
|
|
mir := &updMirror{base: files.URL, sums: map[string]string{
|
|
"abc12345/q4.gguf": "aaaa", "def98765/q4.gguf": "bbbb"}}
|
|
// What was fetched before origins were kept: the mirror alone says where from.
|
|
_, _ = mir.Fetch(context.Background(), mirror.FetchRequest{Model: "qwen", Version: "abc12345", File: "q4.gguf",
|
|
Source: "https://huggingface.co/o/r/resolve/" + oldRev + "/q4.gguf"})
|
|
models := &updModels{models: map[string]catalog.Model{"qwen": {Name: "qwen",
|
|
Source: files.URL + "/qwen/abc12345/q4.gguf",
|
|
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 8}}}}, declared: map[string]bool{}}
|
|
reg := &movingRegistry{repo: hf.Repo{ID: "o/r", Revision: newRev, Files: []hf.File{
|
|
{Path: "q4.gguf", SHA256: "bbbb", Bytes: 2 << 30}, {Path: "README.md", Bytes: 10}}}}
|
|
fleet := &restartFleet{engine: engine.URL, reps: []reconciler.Replica{
|
|
{ID: "qwen-1", Model: "qwen", State: reconciler.Ready, Endpoint: engine.URL},
|
|
{ID: "qwen-2", Model: "qwen", State: reconciler.Ready, Endpoint: engine.URL}}}
|
|
s := New(Deps{Registry: reg, Mirror: mir, Catalog: &fakeCatalog{}, Fleet: fleet,
|
|
Current: models.all, Models: models, Restart: fleet.Restart,
|
|
Poll: 10 * time.Millisecond, ReadyTimeout: 5 * time.Second})
|
|
return s, mir, models, reg, fleet
|
|
}
|
|
|
|
func TestNewWeightsInTheRepositoryAreFoundForAModelServedBeforeOriginsWereKept(t *testing.T) {
|
|
s, _, models, _, _ := updSetup(t)
|
|
s.CheckUpdates(context.Background())
|
|
u := s.Updates()["qwen"]
|
|
if !u.Available || u.Repo != "o/r" || u.Current != oldRev || u.Latest != newRev ||
|
|
len(u.Changed) != 1 || u.Changed[0] != "q4.gguf" || u.Bytes != 2<<30 {
|
|
t.Fatalf("%+v", u)
|
|
}
|
|
if o := models.models["qwen"].Origin; o == nil || o.Repo != "o/r" || o.Files[0].SHA256 != "aaaa" {
|
|
t.Errorf("the origin worked out was not recorded: %+v", o)
|
|
}
|
|
}
|
|
|
|
func TestANewCommitThatLeavesTheWeightsAloneIsNotAnUpdate(t *testing.T) {
|
|
s, mir, _, _, _ := updSetup(t)
|
|
mir.held[0].SHA256 = "bbbb"
|
|
u, err := s.CheckModel(context.Background(), "qwen")
|
|
if err != nil || u.Available || u.Latest != newRev || updateSays(u) != "same" {
|
|
t.Fatalf("%+v %v", u, err)
|
|
}
|
|
}
|
|
|
|
func updateSays(u UpdateInfo) string {
|
|
if !u.Available && u.Latest != u.Current && len(u.Missing) == 0 {
|
|
return "same"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func TestAnUpdateFetchesBesideTheOldReplacesTheReplicasAndCanGoBack(t *testing.T) {
|
|
s, mir, models, _, fleet := updSetup(t)
|
|
oldSource := models.models["qwen"].Source
|
|
job, err := s.StartUpdate("qwen", "tester")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
j := wait(t, s, job.ID)
|
|
if j.State != Done {
|
|
t.Fatalf("%s: %+v", j.Error, j.Steps)
|
|
}
|
|
m := models.models["qwen"]
|
|
if !strings.HasSuffix(m.Source, "/qwen/def98765/q4.gguf") || m.Origin.Revision != newRev ||
|
|
m.Origin.Previous == nil || m.Origin.Previous.Source != oldSource {
|
|
t.Fatalf("catalog: %s %+v", m.Source, m.Origin)
|
|
}
|
|
if len(mir.held) != 2 {
|
|
t.Errorf("the old files did not stay on the mirror beside the new: %+v", mir.held)
|
|
}
|
|
for _, r := range fleet.Replicas() {
|
|
if !strings.Contains(r.ID, "new") {
|
|
t.Errorf("%s was not replaced", r.ID)
|
|
}
|
|
}
|
|
if u := s.Updates()["qwen"]; u.Available || u.Current != newRev {
|
|
t.Errorf("still offered: %+v", u)
|
|
}
|
|
|
|
// And back.
|
|
if _, err := s.Rollback(context.Background(), "qwen", "tester"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m = models.models["qwen"]
|
|
if m.Source != oldSource || m.Origin.Revision != oldRev || m.Origin.Previous.Revision != newRev {
|
|
t.Fatalf("rollback: %s %+v", m.Source, m.Origin)
|
|
}
|
|
if u := s.Updates()["qwen"]; !u.Available {
|
|
t.Errorf("the update it went back from is not offered again: %+v", u)
|
|
}
|
|
|
|
// And forward again, from files already on the mirror.
|
|
job, err = s.StartUpdate("qwen", "tester")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if j := wait(t, s, job.ID); j.State != Done || models.models["qwen"].Origin.Revision != newRev {
|
|
t.Fatalf("%s: %+v", j.Error, j.Steps)
|
|
}
|
|
// Then there is nothing newer: refused, saying why.
|
|
job, _ = s.StartUpdate("qwen", "tester")
|
|
if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "already runs") {
|
|
t.Fatalf("%+v", j)
|
|
}
|
|
}
|
|
|
|
func TestAModelFromConfigurationIsNotUpdatedBehindItsBack(t *testing.T) {
|
|
s, _, models, _, _ := updSetup(t)
|
|
models.declared["qwen"] = true
|
|
if _, err := s.StartUpdate("qwen", "t"); err == nil || !strings.Contains(err.Error(), "server.yaml") {
|
|
t.Fatalf("%v", err)
|
|
}
|
|
}
|
|
|
|
func TestAnUpdateToFilesTheRepositoryDroppedIsRefused(t *testing.T) {
|
|
s, _, _, reg, _ := updSetup(t)
|
|
reg.repo.Files = []hf.File{{Path: "q4-v2.gguf", SHA256: "cccc"}}
|
|
job, _ := s.StartUpdate("qwen", "t")
|
|
if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "no longer has q4.gguf") {
|
|
t.Fatalf("%+v", j)
|
|
}
|
|
}
|
|
|
|
func TestRegistryDownloadURLsAreRead(t *testing.T) {
|
|
for u, want := range map[string][3]string{
|
|
"https://huggingface.co/bartowski/Qwen-GGUF/resolve/abc/sub/f.gguf": {"bartowski/Qwen-GGUF", "abc", "sub/f.gguf"},
|
|
"https://hf.internal/mirror/x/resolve/main/f.gguf": {"mirror/x", "main", "f.gguf"},
|
|
} {
|
|
repo, rev, path, ok := parseResolve(u)
|
|
if !ok || [3]string{repo, rev, path} != want {
|
|
t.Errorf("%s: %s %s %s %v", u, repo, rev, path, ok)
|
|
}
|
|
}
|
|
for _, u := range []string{"https://example.com/f.gguf", "https://huggingface.co/a/b/c/resolve/x/f", ":"} {
|
|
if _, _, _, ok := parseResolve(u); ok {
|
|
t.Errorf("%s was read", u)
|
|
}
|
|
}
|
|
}
|