diff --git a/CHANGELOG.md b/CHANGELOG.md index 341c596..6481394 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,20 @@ their own — a key allowed an adapter does not reach the model under it. Adapte files are kept from weight cleanup like the model's own. Not yet for vLLM or SGLang profiles, whose adapters are loaded differently. +### Finding and attaching adapters from the mirror page + +- **Explore** has a *LoRA adapters* view: adapters for any model, or — from a + held model's card, *find adapters on Hugging Face* — those made for the model + its repository was made from, since a GGUF repository is a quantisation and + adapters are made for the original. +- **Opening a repository** reads the first bytes of each GGUF file and marks it + a *LoRA adapter* or a *model*. Repositories tagged "lora" hold a model with + the adapter merged in about as often as an adapter, and only the file says. +- **Held adapter files** are marked in the list, and a model's card lists its + adapters, offers the held adapter files of its architecture to attach under a + name and scale, and removes them. A file that is not an adapter, or is for + another architecture, is refused before it reaches the catalog. + ### Models served outside the cluster A catalog model can be an OpenAI-compatible API elsewhere: `external: {url, diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go index 984b94c..303653f 100644 --- a/internal/control/modelsvc/modelsvc.go +++ b/internal/control/modelsvc/modelsvc.go @@ -271,6 +271,23 @@ func (s *Service) SetOwner(ctx context.Context, name, owner, by string) (catalog return final, err } +// SetAdapters replaces the LoRA adapters a model serves. Replicas load the +// new set when they next start. +func (s *Service) SetAdapters(ctx context.Context, name string, adapters []catalog.Adapter, by string) (catalog.Model, error) { + final, err := s.update(ctx, name, by, "change its adapters", func(m *catalog.Model) error { + m.Adapters = adapters + return nil + }) + if err == nil { + names := make([]string, 0, len(adapters)) + for _, a := range adapters { + names = append(names, a.Name) + } + s.log.Info("model adapters set", "model", name, "adapters", names, "by", by) + } + return final, err +} + // update changes one model added at runtime and keeps the change. // // Built before anything is written, like Add: catalog.New is what applies the diff --git a/internal/gguf/gguf.go b/internal/gguf/gguf.go index 906a47c..9d88179 100644 --- a/internal/gguf/gguf.go +++ b/internal/gguf/gguf.go @@ -25,6 +25,10 @@ type Header struct { Version uint32 Architecture string Name string + // Type is general.type: "model", or "adapter" for a LoRA adapter, which + // is applied to a model rather than run on its own. Empty in files made + // before the key existed, which are models. + Type string // The shape the KV cache is sized from, from the architecture's own keys // (llama.block_count, qwen2.attention.head_count_kv, ...). Zero when the @@ -235,6 +239,8 @@ func Read(r io.Reader) (Header, error) { h.Architecture = v case "general.name": h.Name = v + case "general.type": + h.Type = v } if h.Architecture != "" { switch strings.TrimPrefix(key, h.Architecture+".") { @@ -290,6 +296,41 @@ func FromURL(ctx context.Context, client *http.Client, url string) (Header, erro return Read(io.LimitReader(resp.Body, headerBytes)) } +// IsAdapter reports whether the file is a LoRA adapter rather than a model. +func (h Header) IsAdapter() bool { return h.Type == "adapter" } + +// adapterProbe is how much of a file is read to tell an adapter from a model: +// general.type comes before the tokenizer, which is what makes a model's +// header megabytes long. +const adapterProbe = 256 << 10 + +// KindFromURL reads just enough of a GGUF file to say what it is — its type +// and architecture — over HTTP. A header cut short by the probe is fine: the +// keys asked for come first. +func KindFromURL(ctx context.Context, client *http.Client, url string) (Header, error) { + if client == nil { + client = &http.Client{Timeout: 30 * time.Second} + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return Header{}, err + } + req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", adapterProbe-1)) + resp, err := client.Do(req) + if err != nil { + return Header{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + return Header{}, fmt.Errorf("%s answered %s", url, resp.Status) + } + h, err := Read(io.LimitReader(resp.Body, adapterProbe)) + if h.Architecture != "" { + return h, nil // what was asked for was read, whatever came after + } + return h, err +} + // IsGGUF reports whether a source names a GGUF file, or a directory of them. func IsGGUF(source string) bool { s := strings.ToLower(strings.TrimSuffix(source, "/")) diff --git a/internal/gguf/kind_live_test.go b/internal/gguf/kind_live_test.go new file mode 100644 index 0000000..33a11f1 --- /dev/null +++ b/internal/gguf/kind_live_test.go @@ -0,0 +1,30 @@ +package gguf + +import ( + "os" + "testing" +) + +// Run by hand against the real adapter this was written for: +// LLMGW_LIVE_GGUF=1 go test ./internal/gguf -run LiveAdapter -v +func TestLiveAdapterIsAnAdapter(t *testing.T) { + if os.Getenv("LLMGW_LIVE_GGUF") == "" { + t.Skip("set LLMGW_LIVE_GGUF to read a real adapter from Hugging Face") + } + h, err := KindFromURL(t.Context(), nil, + "https://huggingface.co/ggml-org/LoRA-Qwen2.5-Coder-7B-Instruct-F16-GGUF/resolve/main/LoRA-Qwen2.5-Coder-7B-Instruct-f16.gguf") + if err != nil { + t.Fatal(err) + } + if !h.IsAdapter() || h.Architecture != "qwen2" { + t.Errorf("%+v", h) + } + m, err := KindFromURL(t.Context(), nil, + "https://huggingface.co/bartowski/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/Qwen2.5-0.5B-Instruct-Q4_K_M.gguf") + if err != nil { + t.Fatal(err) + } + if m.IsAdapter() || m.Architecture != "qwen2" { + t.Errorf("a model read as %+v", m) + } +} diff --git a/internal/gguf/kind_test.go b/internal/gguf/kind_test.go new file mode 100644 index 0000000..ad40182 --- /dev/null +++ b/internal/gguf/kind_test.go @@ -0,0 +1,58 @@ +package gguf + +import ( + "bytes" + "encoding/binary" + "net/http" + "net/http/httptest" + "testing" +) + +// adapterHeader is the start of a LoRA adapter as llama.cpp's converter +// writes it, followed by padding that stands in for its tensors. +func adapterHeader(kind string, pad int) []byte { + var b bytes.Buffer + w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) } + str := func(s string) { w(uint64(len(s))); b.WriteString(s) } + b.WriteString("GGUF") + w(uint32(3)) + w(uint64(0)) + w(uint64(3)) + str("general.architecture") + w(uint32(tString)) + str("qwen2") + str("general.type") + w(uint32(tString)) + str(kind) + str("adapter.type") + w(uint32(tString)) + str("lora") + b.Write(make([]byte, pad)) + return b.Bytes() +} + +func TestAnAdapterIsToldFromAModelByItsFirstBytes(t *testing.T) { + var asked string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + asked = r.Header.Get("Range") + w.WriteHeader(http.StatusPartialContent) + if r.URL.Path == "/adapter.gguf" { + _, _ = w.Write(adapterHeader("adapter", 1<<20)) + return + } + _, _ = w.Write(header("qwen2")) + })) + defer srv.Close() + + h, err := KindFromURL(t.Context(), srv.Client(), srv.URL+"/adapter.gguf") + if err != nil || !h.IsAdapter() || h.Architecture != "qwen2" { + t.Errorf("an adapter read as %+v, %v", h, err) + } + if asked != "bytes=0-262143" { + t.Errorf("asked for %q: the probe should read the start only", asked) + } + m, err := KindFromURL(t.Context(), srv.Client(), srv.URL+"/model.gguf") + if err != nil || m.IsAdapter() || m.Architecture != "qwen2" { + t.Errorf("a model read as %+v, %v", m, err) + } +} diff --git a/internal/hf/hf.go b/internal/hf/hf.go index 65a2bb4..cd8e647 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -156,6 +156,10 @@ type Query struct { // applies it, so a page of small models is a full page. MaxParams int64 Limit int + // Filters are further registry tags every result must carry, such as + // "lora", or "base_model:adapter:Qwen/Qwen2.5-7B-Instruct" for the + // adapters made for one model. + Filters []string } // exploreFields is everything a listing shows. Asking for any field with @@ -188,7 +192,10 @@ func (c *Client) Explore(ctx context.Context, q Query) ([]Model, error) { v.Set("pipeline_tag", q.Task) } if q.GGUF { - v.Set("filter", "gguf") + v.Add("filter", "gguf") + } + for _, f := range q.Filters { + v.Add("filter", f) } if q.MaxParams > 0 { v.Set("num_parameters", fmt.Sprintf("min:0,max:%d", q.MaxParams)) diff --git a/internal/web/account.go b/internal/web/account.go index e24a6e8..6679c16 100644 --- a/internal/web/account.go +++ b/internal/web/account.go @@ -9,6 +9,7 @@ import ( "net/http" "strconv" "strings" + "sync" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/account" @@ -76,6 +77,9 @@ type AccountPages struct { priced Priced // registry browses a model registry, or is nil where there is no way out. registry Registry + // kinds remembers what each held GGUF file is, by its address: a held + // version never changes, so it is read once. + kinds sync.Map // catalog puts a mirrored model into service; fleet says whether anything // could run it. Both nil where the mirror page is read-only. catalog CatalogAdmin diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index 0402396..611615b 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -11,6 +11,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" @@ -46,6 +47,10 @@ type Registry interface { // repoSet is one thing worth fetching: a file, or every shard of one. type repoSet struct { + // Kind is what the file says it is, read from its first bytes: "adapter" + // for a LoRA adapter, "model", or empty when it was not read. Arch is the + // architecture it names, which an adapter must share with its model. + Kind, Arch string // Verdict is whether the nodes' llama.cpp can load it, from the // architecture the registry reports; a set that cannot be served is // shown with the reason and no fetch button. @@ -76,6 +81,8 @@ type modelUse struct { // a replica of its first profile reserves. Class, EffClass, Owner string MemoryGB, CPUCores int + // Adapters are the LoRA adapters it serves. + Adapters []catalog.Adapter // Ready, Wanted and Nodes are how it runs now, for the row itself: // replicas ready of those it should have, and the nodes they are on. // Fleet is false where nothing could be asked, so the row says nothing @@ -105,6 +112,9 @@ type mirrorRow struct { // Detail is the card opened beneath the row: what runs it, on which // nodes, and how many replicas it runs. Detail *heldDetail + // Kind is what a single GGUF file is — "adapter" or "model" — and Arch + // its architecture; empty when not read. + Kind, Arch string // Open is the row's ?open= value. Open string @@ -299,6 +309,7 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) { })) mux.HandleFunc("POST /admin/mirror/replicas", p.mirrorAction(m, p.setModel)) + mux.HandleFunc("POST /admin/mirror/adapters", p.mirrorAction(m, p.setAdapters)) mux.HandleFunc("POST /admin/mirror/serve", p.mirrorAction(m, func(ctx context.Context, admin store.User, r *http.Request) (string, error) { @@ -469,6 +480,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request, Name: m.Name, Mode: modeOf(m.Scaling), Replicas: max(m.Scaling.Max, 1), Min: m.Scaling.Min, Max: m.Scaling.Max, Class: m.Class, EffClass: m.EffectiveClass(), Owner: m.Owner, + Adapters: m.Adapters, Declared: p.catalog.Declared(m.Name), } if len(m.Profiles) > 0 { @@ -487,6 +499,8 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request, } } + p.heldKinds(r.Context(), d.Rows) + // The one row whose card is open. From the form too, so a change made in // the card comes back with the card still open to show it. open := r.FormValue("open") @@ -494,6 +508,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request, d.Rows[i].Open = openKey(d.Rows[i].Model, d.Rows[i].Version) if open != "" && d.Rows[i].Open == open && d.Rows[i].State == mirror.Present { d.Rows[i].Detail = p.heldDetailFor(r.Context(), d.Rows[i]) + d.Rows[i].Detail.addAdapterChoices(d.Rows[i], d.Rows) } } @@ -755,6 +770,46 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string } d.RepoSets = append(d.RepoSets, rs) } + p.probeKinds(r.Context(), d.RepoSets) +} + +// kindOf reads what a GGUF file is; a variable so tests need no registry. +var kindOf = func(ctx context.Context, url string) (gguf.Header, error) { + return gguf.KindFromURL(ctx, nil, url) +} + +// probeKinds reads the start of each single-file GGUF set, to tell LoRA +// adapters from models: a repository tagged "lora" holds a model with the +// adapter merged in as often as an adapter, and only the file says which. +// A few at once and a few in all, since each is a request to the registry. +func (p *AccountPages) probeKinds(ctx context.Context, sets []repoSet) { + const most, together = 12, 6 + sem := make(chan struct{}, together) + var wg sync.WaitGroup + n := 0 + for i := range sets { + s := &sets[i] + if len(s.Sources) != 1 || !gguf.IsGGUF(s.Sources[0].Path) || n == most { + continue + } + n++ + wg.Add(1) + go func() { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + cctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if h, err := kindOf(cctx, s.Sources[0].URL); err == nil { + s.Arch = h.Architecture + s.Kind = "model" + if h.IsAdapter() { + s.Kind = "adapter" + } + } + }() + } + wg.Wait() } // defaultModelName is the catalog name a repository suggests. diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index 9e2ca12..a154b8a 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -27,6 +27,7 @@ type CatalogAdmin interface { SetClass(ctx context.Context, name, class, by string) (catalog.Model, error) SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error) Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error) + SetAdapters(ctx context.Context, name string, adapters []catalog.Adapter, by string) (catalog.Model, error) } // serveForm is the offer to put one mirrored version into service. diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go index f64a454..a427426 100644 --- a/internal/web/admin_serve_test.go +++ b/internal/web/admin_serve_test.go @@ -16,15 +16,16 @@ import ( ) type fakeCatalog struct { - added []catalog.Model - declared string - sources map[string][]catalog.Model - scaled []string - lastScale catalog.Scaling - err error - classes []string - owners []string - resources []string + added []catalog.Model + declared string + sources map[string][]catalog.Model + scaled []string + lastScale catalog.Scaling + err error + classes []string + owners []string + resources []string + adapterSets [][]catalog.Adapter } func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error { @@ -39,6 +40,11 @@ func (f *fakeCatalog) Sources(context.Context) (map[string][]catalog.Model, erro return f.sources, nil } +func (f *fakeCatalog) SetAdapters(_ context.Context, name string, as []catalog.Adapter, _ string) (catalog.Model, error) { + f.adapterSets = append(f.adapterSets, as) + return catalog.Model{Name: name, Adapters: as}, f.err +} + func (f *fakeCatalog) SetClass(_ context.Context, name, class, _ string) (catalog.Model, error) { f.classes = append(f.classes, name+"="+class) return catalog.Model{Name: name, Class: class}, f.err diff --git a/internal/web/mirror_detail.go b/internal/web/mirror_detail.go index 7c3136c..98dadfa 100644 --- a/internal/web/mirror_detail.go +++ b/internal/web/mirror_detail.go @@ -4,13 +4,19 @@ import ( "context" "errors" "fmt" + "html/template" "net/http" + "net/url" + "slices" "strconv" "strings" + "sync" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -37,9 +43,15 @@ type heldDetail struct { // heldModel is one catalog model running a held version. type heldModel struct { modelUse - View adminapi.ModelView - Replicas []heldReplica - Headroom *adminapi.HeadroomView + // Adapters are the model's own; AdapterChoices the held adapter files + // of its architecture not attached yet; FindAdapters the explorer's + // list of adapters made for it. + Adapters []catalog.Adapter + AdapterChoices []adapterChoice + FindAdapters template.URL + View adminapi.ModelView + Replicas []heldReplica + Headroom *adminapi.HeadroomView // MostNow is how many replicas could always run, Current plus the room // found; zero when the room was not measured or had no end in sight. MostNow int @@ -74,7 +86,7 @@ func (p *AccountPages) heldDetailFor(ctx context.Context, row mirrorRow) *heldDe } if p.fleet == nil { for _, u := range row.UsedBy { - d.Models = append(d.Models, heldModel{modelUse: u}) + d.Models = append(d.Models, heldModel{modelUse: u, Adapters: u.Adapters}) } return d } @@ -88,7 +100,7 @@ func (p *AccountPages) heldDetailFor(ctx context.Context, row mirrorRow) *heldDe } hs, _ := p.fleet.(HeadroomSource) for _, u := range row.UsedBy { - hm := heldModel{modelUse: u, View: views[u.Name]} + hm := heldModel{modelUse: u, View: views[u.Name], Adapters: u.Adapters} for _, rep := range p.fleet.Replicas(u.Name) { // Built field by field: a ReplicaView carries the engine's key, // which is for the admin listener and never for a page. @@ -282,3 +294,140 @@ func (p *AccountPages) setModel(ctx context.Context, admin store.User, r *http.R } return name + " " + strings.Join(done, "; "), nil } + +// heldKinds reads what each held single-file GGUF is — an adapter or a model — +// once per file, remembered after. +func (p *AccountPages) heldKinds(ctx context.Context, rows []mirrorRow) { + var wg sync.WaitGroup + sem := make(chan struct{}, 6) + for i := range rows { + row := &rows[i] + if row.State != mirror.Present || row.Files != 1 || !gguf.IsGGUF(row.File) { + continue + } + if h, ok := p.kinds.Load(row.URL); ok { + row.Kind, row.Arch = kindName(h.(gguf.Header)), h.(gguf.Header).Architecture + continue + } + wg.Add(1) + go func() { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + cctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if h, err := kindOf(cctx, row.URL); err == nil { + p.kinds.Store(row.URL, h) + row.Kind, row.Arch = kindName(h), h.Architecture + } + }() + } + wg.Wait() +} + +func kindName(h gguf.Header) string { + if h.IsAdapter() { + return "adapter" + } + return "model" +} + +// adapterChoice is a held adapter file that could be attached to a model. +type adapterChoice struct { + URL, Label, Suggest string +} + +// addAdapterChoices offers, on each model of a held model's card, the held +// adapter files of the same architecture, and a way to find more. +func (d *heldDetail) addAdapterChoices(row mirrorRow, rows []mirrorRow) { + for i := range d.Models { + hm := &d.Models[i] + hm.FindAdapters = findAdaptersHref(row.Source) + if row.Kind == "adapter" { + continue + } + for _, r := range rows { + if r.Kind != "adapter" || (row.Arch != "" && r.Arch != row.Arch) { + continue + } + attached := false + for _, a := range hm.Adapters { + attached = attached || a.Source == r.URL + } + if !attached { + hm.AdapterChoices = append(hm.AdapterChoices, adapterChoice{URL: r.URL, + Label: r.Model + " " + r.Version + " · " + r.File, Suggest: hm.Name + "-" + r.Model}) + } + } + } +} + +// findAdaptersHref is the explorer's list of adapters for a held model: those +// made for the model its repository was made from, when it came from the +// registry, and every adapter otherwise. +func findAdaptersHref(source string) template.URL { + const base = "/admin/mirror?tab=explore&view=adapters" + u, err := url.Parse(source) + if err != nil || u.Host != "huggingface.co" { + return base + } + parts := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(parts) < 2 { + return base + } + return template.URL(base + "&from=" + url.QueryEscape(parts[0]+"/"+parts[1])) +} + +// setAdapters attaches or detaches one adapter on a model, from its card. +// An adapter is checked before it is attached: a file that is not an adapter, +// or one made for another architecture, is a replica that fails to start. +func (p *AccountPages) setAdapters(ctx context.Context, admin store.User, r *http.Request) (string, error) { + name := r.FormValue("model") + current, err := p.currentModel(ctx, name) + if err != nil { + return "", err + } + adapters := slices.Clone(current.Adapters) + switch r.FormValue("op") { + case "remove": + gone := r.FormValue("name") + adapters = slices.DeleteFunc(adapters, func(a catalog.Adapter) bool { return a.Name == gone }) + if len(adapters) == len(current.Adapters) { + return "", fmt.Errorf("%s has no adapter %q", name, gone) + } + if _, err := p.catalog.SetAdapters(ctx, name, adapters, admin.Email); err != nil { + return "", err + } + return fmt.Sprintf("%s: adapter %s removed; replicas drop it when restarted (llmgw ctl restart --model %s)", + name, gone, name), nil + case "add": + an := strings.TrimSpace(r.FormValue("name")) + src := strings.TrimSpace(r.FormValue("source")) + scale := 1.0 + if v := strings.TrimSpace(r.FormValue("scale")); v != "" { + if scale, err = strconv.ParseFloat(v, 64); err != nil || scale <= 0 { + return "", errors.New("the scale is a positive number") + } + } + if an == "" || src == "" { + return "", errors.New("an adapter needs a name and a file") + } + h, err := kindOf(ctx, src) + if err != nil { + return "", fmt.Errorf("the adapter file could not be read: %w", err) + } + if !h.IsAdapter() { + return "", errors.New("that file is a model, not a LoRA adapter") + } + if base, err := kindOf(ctx, current.Source); err == nil && base.Architecture != h.Architecture { + return "", fmt.Errorf("that adapter is for %s models and %s is %s", h.Architecture, name, base.Architecture) + } + adapters = append(adapters, catalog.Adapter{Name: an, Source: src, Scale: scale}) + if _, err := p.catalog.SetAdapters(ctx, name, adapters, admin.Email); err != nil { + return "", err + } + return fmt.Sprintf("%s: adapter %s added; ask for it by that name once the replicas are restarted "+ + "(llmgw ctl restart --model %s)", name, an, name), nil + } + return "", errors.New("add or remove an adapter") +} diff --git a/internal/web/mirror_detail_test.go b/internal/web/mirror_detail_test.go index daa99b9..0d6eee8 100644 --- a/internal/web/mirror_detail_test.go +++ b/internal/web/mirror_detail_test.go @@ -1,6 +1,7 @@ package web import ( + "context" "log/slog" "net/http" "net/url" @@ -9,6 +10,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -269,3 +271,65 @@ func TestRoomAHigherClassWouldMakeIsOfferedAndSaid(t *testing.T) { t.Errorf("three is past even what preemption makes: %s", body) } } + +func TestAnAdapterHeldInTheMirrorIsAttachedFromItsModelsCard(t *testing.T) { + prev := kindOf + t.Cleanup(func() { kindOf = prev }) + kindOf = func(_ context.Context, u string) (gguf.Header, error) { + switch { + case strings.Contains(u, "/sql/"): + return gguf.Header{Architecture: "qwen2", Type: "adapter"}, nil + case strings.Contains(u, "/llama-lora/"): + return gguf.Header{Architecture: "llama", Type: "adapter"}, nil + } + return gguf.Header{Architecture: "qwen2"}, nil + } + cat := coderCatalog() + coder := &cat.sources["http://mirror:8081/coder/v1/c.gguf"][0] + coder.Source = "http://mirror:8081/coder/v1/c.gguf" + coder.Profiles = []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 12}} + m := &fakeMirror{models: []mirror.Status{ + {Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 1, + Source: "https://huggingface.co/bartowski/Qwen2.5-Coder-7B-Instruct-GGUF/resolve/abc/c.gguf"}, + {Model: "sql", Version: "v1", File: "sql.gguf", State: mirror.Present, BytesDone: 1}, + {Model: "llama-lora", Version: "v1", File: "l.gguf", State: mirror.Present, BytesDone: 1}, + }} + mux := mirrorPageDeps(t, MirrorDeps{Mirror: m, Catalog: cat, Fleet: &roomyFleet{more: 1}}, true) + + list := mirrorGet(t, mux, "/admin/mirror").Body.String() + if strings.Count(list, ">LoRA adapter") != 2 { + t.Error("the two adapter files are not marked as adapters in the list") + } + card := mirrorGet(t, mux, "/admin/mirror?open=coder:v1").Body.String() + if !strings.Contains(card, `
{{range .Rows}}{{$row0 := .}}{{.File}}{{end}}{{if .Of}}LoRA adapters made for {{.Of}} ·
+ all adapters{{else}}LoRA adapters for any model.
+ From a held model's card, find adapters lists those made for it.{{end}}
+ An adapter runs on the model it was made for, and only as a GGUF adapter file;
+ open one to see which of its files are.