package mirror import ( "net/http" "net/http/httptest" "os" "path/filepath" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" ) // testMirror is a mirror with its file server and management API, as a peer // reaches it. type testMirror struct { root string m *Manager files *httptest.Server api *httptest.Server } func newTestMirror(t *testing.T) *testMirror { t.Helper() root := t.TempDir() tm := &testMirror{root: root, m: NewManager(root, discardLog())} tm.m.AllowPrivateSources() tm.files = httptest.NewServer(http.FileServer(http.Dir(root))) tm.api = httptest.NewServer(NewAPI(tm.m, tm.files.URL).Handler()) t.Cleanup(func() { tm.files.Close(); tm.api.Close() }) return tm } func (tm *testMirror) client(t *testing.T) *Client { c, err := NewClient(tm.api.URL, tlsconf.Config{}, 5*time.Second) if err != nil { t.Fatal(err) } return c } func (tm *testMirror) has(model, version string) bool { st, ok := tm.m.Status(model, version) return ok && st.State == Present } func waitPresent(t *testing.T, tm *testMirror, model, version string) { t.Helper() deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { if tm.has(model, version) { return } time.Sleep(20 * time.Millisecond) } st, _ := tm.m.Status(model, version) t.Fatalf("%s/%s never arrived: %+v", model, version, st) } func TestTwoMirrorsKeepInStepAndAgreeOnWhatWasRemoved(t *testing.T) { blob := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("weights of a small model")) })) defer blob.Close() a, b := newTestMirror(t), newTestMirror(t) if _, err := a.m.Start(blob.URL+"/m.gguf", Entry{Model: "qwen", Version: "v1", File: "m.gguf"}); err != nil { t.Fatal(err) } waitPresent(t, a, "qwen", "v1") // b copies what a has, its checksum checked. pb := &Peers{M: b.m, Log: discardLog()} pb.SyncOnce(t.Context(), a.client(t)) waitPresent(t, b, "qwen", "v1") got, _ := os.ReadFile(filepath.Join(b.root, "qwen", "v1", "m.gguf")) if string(got) != "weights of a small model" { t.Errorf("b holds %q", got) } // Removed on a: a does not copy it back from b, and b removes it too. time.Sleep(10 * time.Millisecond) if _, err := a.m.RemoveVersion("qwen", "v1"); err != nil { t.Fatal(err) } pa := &Peers{M: a.m, Log: discardLog()} pa.SyncOnce(t.Context(), b.client(t)) time.Sleep(200 * time.Millisecond) if a.has("qwen", "v1") { t.Fatal("a copied back from b what it had removed") } pb.SyncOnce(t.Context(), a.client(t)) if b.has("qwen", "v1") { t.Error("b kept what a removed") } // Added again on a, after the removal: the new one stands, on both. time.Sleep(10 * time.Millisecond) if _, err := a.m.Start(blob.URL+"/m.gguf", Entry{Model: "qwen", Version: "v1", File: "m.gguf"}); err != nil { t.Fatal(err) } waitPresent(t, a, "qwen", "v1") pb.SyncOnce(t.Context(), a.client(t)) waitPresent(t, b, "qwen", "v1") } // A file that is not what the peer says it is, is not kept. func TestACopyWhoseChecksumIsNotThePeersIsNotKept(t *testing.T) { served := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("something else")) })) defer served.Close() b := newTestMirror(t) e := Entry{Model: "qwen", Version: "v1", File: "m.gguf"} if _, err := b.m.startCopy(served.URL+"/m.gguf", e, "0000"); err != nil { t.Fatal(err) } deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { if st, _ := b.m.Status("qwen", "v1"); st.State == Failed { break } time.Sleep(20 * time.Millisecond) } if _, err := os.Stat(e.Path(b.root)); !os.IsNotExist(err) { t.Errorf("a copy with the wrong checksum was kept: %v", err) } } // A mirror that cannot be reached is passed over for one kept in step with it. func TestTheNextMirrorIsAskedWhenTheFirstCannotBeReached(t *testing.T) { a, b := newTestMirror(t), newTestMirror(t) c, err := NewFailoverClient([]string{a.api.URL, b.api.URL}, tlsconf.Config{}, 2*time.Second) if err != nil { t.Fatal(err) } if list, err := c.List(t.Context()); err != nil || list.Base != a.files.URL { t.Fatalf("both up: %+v %v", list, err) } a.api.Close() if list, err := c.List(t.Context()); err != nil || list.Base != b.files.URL { t.Errorf("the first down: %+v %v", list, err) } b.api.Close() if _, err := c.List(t.Context()); err == nil { t.Error("both down, and an answer") } } // A node is given a model's file on a mirror that answers. func TestAModelsAddressIsMovedToAMirrorThatAnswers(t *testing.T) { a, b := newTestMirror(t), newTestMirror(t) s, err := NewSources([]string{a.api.URL, b.api.URL}, tlsconf.Config{}, discardLog()) if err != nil { t.Fatal(err) } s.Check(t.Context()) src := a.files.URL + "/qwen/v1/m.gguf" if got := s.Rewrite(src); got != src { t.Errorf("its own mirror up: %s", got) } a.api.Close() s.Check(t.Context()) if got := s.Rewrite(src); got != b.files.URL+"/qwen/v1/m.gguf" { t.Errorf("its own mirror down: %s", got) } if got := s.Rewrite("https://huggingface.co/x/y"); got != "https://huggingface.co/x/y" { t.Errorf("an address on no mirror: %s", got) } b.api.Close() s.Check(t.Context()) if got := s.Rewrite(src); got != src { t.Errorf("none up: %s", got) } }