Find LoRA adapters on Hugging Face and attach them from the mirror page
An adapters view in the explorer, filtered by the model a held model was made from; GGUF files told apart as adapter or model by their first bytes; held adapter files marked; and a model's card attaches and detaches held adapters of its architecture, refusing models and mismatched architectures. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
cf53904cc5
commit
98bc03a073
16 files changed
+609
-19
No files matched your search
@@ -54,6 +54,20 @@ their own — a key allowed an adapter does not reach the model under it. Adapte
|
||||
files are kept from weight cleanup like the model's own. Not yet for vLLM or
|
||||
SGLang profiles, whose adapters are loaded differently.
|
||||
|
||||
### Finding and attaching adapters from the mirror page
|
||||
|
||||
- **Explore** has a *LoRA adapters* view: adapters for any model, or — from a
|
||||
held model's card, *find adapters on Hugging Face* — those made for the model
|
||||
its repository was made from, since a GGUF repository is a quantisation and
|
||||
adapters are made for the original.
|
||||
- **Opening a repository** reads the first bytes of each GGUF file and marks it
|
||||
a *LoRA adapter* or a *model*. Repositories tagged "lora" hold a model with
|
||||
the adapter merged in about as often as an adapter, and only the file says.
|
||||
- **Held adapter files** are marked in the list, and a model's card lists its
|
||||
adapters, offers the held adapter files of its architecture to attach under a
|
||||
name and scale, and removes them. A file that is not an adapter, or is for
|
||||
another architecture, is refused before it reaches the catalog.
|
||||
|
||||
### Models served outside the cluster
|
||||
|
||||
A catalog model can be an OpenAI-compatible API elsewhere: `external: {url,
|
||||
|
||||
@@ -271,6 +271,23 @@ func (s *Service) SetOwner(ctx context.Context, name, owner, by string) (catalog
|
||||
return final, err
|
||||
}
|
||||
|
||||
// SetAdapters replaces the LoRA adapters a model serves. Replicas load the
|
||||
// new set when they next start.
|
||||
func (s *Service) SetAdapters(ctx context.Context, name string, adapters []catalog.Adapter, by string) (catalog.Model, error) {
|
||||
final, err := s.update(ctx, name, by, "change its adapters", func(m *catalog.Model) error {
|
||||
m.Adapters = adapters
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
names := make([]string, 0, len(adapters))
|
||||
for _, a := range adapters {
|
||||
names = append(names, a.Name)
|
||||
}
|
||||
s.log.Info("model adapters set", "model", name, "adapters", names, "by", by)
|
||||
}
|
||||
return final, err
|
||||
}
|
||||
|
||||
// update changes one model added at runtime and keeps the change.
|
||||
//
|
||||
// Built before anything is written, like Add: catalog.New is what applies the
|
||||
|
||||
@@ -25,6 +25,10 @@ type Header struct {
|
||||
Version uint32
|
||||
Architecture string
|
||||
Name string
|
||||
// Type is general.type: "model", or "adapter" for a LoRA adapter, which
|
||||
// is applied to a model rather than run on its own. Empty in files made
|
||||
// before the key existed, which are models.
|
||||
Type string
|
||||
|
||||
// The shape the KV cache is sized from, from the architecture's own keys
|
||||
// (llama.block_count, qwen2.attention.head_count_kv, ...). Zero when the
|
||||
@@ -235,6 +239,8 @@ func Read(r io.Reader) (Header, error) {
|
||||
h.Architecture = v
|
||||
case "general.name":
|
||||
h.Name = v
|
||||
case "general.type":
|
||||
h.Type = v
|
||||
}
|
||||
if h.Architecture != "" {
|
||||
switch strings.TrimPrefix(key, h.Architecture+".") {
|
||||
@@ -290,6 +296,41 @@ func FromURL(ctx context.Context, client *http.Client, url string) (Header, erro
|
||||
return Read(io.LimitReader(resp.Body, headerBytes))
|
||||
}
|
||||
|
||||
// IsAdapter reports whether the file is a LoRA adapter rather than a model.
|
||||
func (h Header) IsAdapter() bool { return h.Type == "adapter" }
|
||||
|
||||
// adapterProbe is how much of a file is read to tell an adapter from a model:
|
||||
// general.type comes before the tokenizer, which is what makes a model's
|
||||
// header megabytes long.
|
||||
const adapterProbe = 256 << 10
|
||||
|
||||
// KindFromURL reads just enough of a GGUF file to say what it is — its type
|
||||
// and architecture — over HTTP. A header cut short by the probe is fine: the
|
||||
// keys asked for come first.
|
||||
func KindFromURL(ctx context.Context, client *http.Client, url string) (Header, error) {
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 30 * time.Second}
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return Header{}, err
|
||||
}
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", adapterProbe-1))
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return Header{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
return Header{}, fmt.Errorf("%s answered %s", url, resp.Status)
|
||||
}
|
||||
h, err := Read(io.LimitReader(resp.Body, adapterProbe))
|
||||
if h.Architecture != "" {
|
||||
return h, nil // what was asked for was read, whatever came after
|
||||
}
|
||||
return h, err
|
||||
}
|
||||
|
||||
// IsGGUF reports whether a source names a GGUF file, or a directory of them.
|
||||
func IsGGUF(source string) bool {
|
||||
s := strings.ToLower(strings.TrimSuffix(source, "/"))
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package gguf
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Run by hand against the real adapter this was written for:
|
||||
// LLMGW_LIVE_GGUF=1 go test ./internal/gguf -run LiveAdapter -v
|
||||
func TestLiveAdapterIsAnAdapter(t *testing.T) {
|
||||
if os.Getenv("LLMGW_LIVE_GGUF") == "" {
|
||||
t.Skip("set LLMGW_LIVE_GGUF to read a real adapter from Hugging Face")
|
||||
}
|
||||
h, err := KindFromURL(t.Context(), nil,
|
||||
"https://huggingface.co/ggml-org/LoRA-Qwen2.5-Coder-7B-Instruct-F16-GGUF/resolve/main/LoRA-Qwen2.5-Coder-7B-Instruct-f16.gguf")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !h.IsAdapter() || h.Architecture != "qwen2" {
|
||||
t.Errorf("%+v", h)
|
||||
}
|
||||
m, err := KindFromURL(t.Context(), nil,
|
||||
"https://huggingface.co/bartowski/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/Qwen2.5-0.5B-Instruct-Q4_K_M.gguf")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.IsAdapter() || m.Architecture != "qwen2" {
|
||||
t.Errorf("a model read as %+v", m)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package gguf
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// adapterHeader is the start of a LoRA adapter as llama.cpp's converter
|
||||
// writes it, followed by padding that stands in for its tensors.
|
||||
func adapterHeader(kind string, pad int) []byte {
|
||||
var b bytes.Buffer
|
||||
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
||||
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
||||
b.WriteString("GGUF")
|
||||
w(uint32(3))
|
||||
w(uint64(0))
|
||||
w(uint64(3))
|
||||
str("general.architecture")
|
||||
w(uint32(tString))
|
||||
str("qwen2")
|
||||
str("general.type")
|
||||
w(uint32(tString))
|
||||
str(kind)
|
||||
str("adapter.type")
|
||||
w(uint32(tString))
|
||||
str("lora")
|
||||
b.Write(make([]byte, pad))
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func TestAnAdapterIsToldFromAModelByItsFirstBytes(t *testing.T) {
|
||||
var asked string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
asked = r.Header.Get("Range")
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
if r.URL.Path == "/adapter.gguf" {
|
||||
_, _ = w.Write(adapterHeader("adapter", 1<<20))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(header("qwen2"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
h, err := KindFromURL(t.Context(), srv.Client(), srv.URL+"/adapter.gguf")
|
||||
if err != nil || !h.IsAdapter() || h.Architecture != "qwen2" {
|
||||
t.Errorf("an adapter read as %+v, %v", h, err)
|
||||
}
|
||||
if asked != "bytes=0-262143" {
|
||||
t.Errorf("asked for %q: the probe should read the start only", asked)
|
||||
}
|
||||
m, err := KindFromURL(t.Context(), srv.Client(), srv.URL+"/model.gguf")
|
||||
if err != nil || m.IsAdapter() || m.Architecture != "qwen2" {
|
||||
t.Errorf("a model read as %+v, %v", m, err)
|
||||
}
|
||||
}
|
||||
+8
-1
@@ -156,6 +156,10 @@ type Query struct {
|
||||
// applies it, so a page of small models is a full page.
|
||||
MaxParams int64
|
||||
Limit int
|
||||
// Filters are further registry tags every result must carry, such as
|
||||
// "lora", or "base_model:adapter:Qwen/Qwen2.5-7B-Instruct" for the
|
||||
// adapters made for one model.
|
||||
Filters []string
|
||||
}
|
||||
|
||||
// exploreFields is everything a listing shows. Asking for any field with
|
||||
@@ -188,7 +192,10 @@ func (c *Client) Explore(ctx context.Context, q Query) ([]Model, error) {
|
||||
v.Set("pipeline_tag", q.Task)
|
||||
}
|
||||
if q.GGUF {
|
||||
v.Set("filter", "gguf")
|
||||
v.Add("filter", "gguf")
|
||||
}
|
||||
for _, f := range q.Filters {
|
||||
v.Add("filter", f)
|
||||
}
|
||||
if q.MaxParams > 0 {
|
||||
v.Set("num_parameters", fmt.Sprintf("min:0,max:%d", q.MaxParams))
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/account"
|
||||
@@ -76,6 +77,9 @@ type AccountPages struct {
|
||||
priced Priced
|
||||
// registry browses a model registry, or is nil where there is no way out.
|
||||
registry Registry
|
||||
// kinds remembers what each held GGUF file is, by its address: a held
|
||||
// version never changes, so it is read once.
|
||||
kinds sync.Map
|
||||
// catalog puts a mirrored model into service; fleet says whether anything
|
||||
// could run it. Both nil where the mirror page is read-only.
|
||||
catalog CatalogAdmin
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
@@ -46,6 +47,10 @@ type Registry interface {
|
||||
|
||||
// repoSet is one thing worth fetching: a file, or every shard of one.
|
||||
type repoSet struct {
|
||||
// Kind is what the file says it is, read from its first bytes: "adapter"
|
||||
// for a LoRA adapter, "model", or empty when it was not read. Arch is the
|
||||
// architecture it names, which an adapter must share with its model.
|
||||
Kind, Arch string
|
||||
// Verdict is whether the nodes' llama.cpp can load it, from the
|
||||
// architecture the registry reports; a set that cannot be served is
|
||||
// shown with the reason and no fetch button.
|
||||
@@ -76,6 +81,8 @@ type modelUse struct {
|
||||
// a replica of its first profile reserves.
|
||||
Class, EffClass, Owner string
|
||||
MemoryGB, CPUCores int
|
||||
// Adapters are the LoRA adapters it serves.
|
||||
Adapters []catalog.Adapter
|
||||
// Ready, Wanted and Nodes are how it runs now, for the row itself:
|
||||
// replicas ready of those it should have, and the nodes they are on.
|
||||
// Fleet is false where nothing could be asked, so the row says nothing
|
||||
@@ -105,6 +112,9 @@ type mirrorRow struct {
|
||||
// Detail is the card opened beneath the row: what runs it, on which
|
||||
// nodes, and how many replicas it runs.
|
||||
Detail *heldDetail
|
||||
// Kind is what a single GGUF file is — "adapter" or "model" — and Arch
|
||||
// its architecture; empty when not read.
|
||||
Kind, Arch string
|
||||
// Open is the row's ?open= value.
|
||||
Open string
|
||||
|
||||
@@ -299,6 +309,7 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
|
||||
}))
|
||||
|
||||
mux.HandleFunc("POST /admin/mirror/replicas", p.mirrorAction(m, p.setModel))
|
||||
mux.HandleFunc("POST /admin/mirror/adapters", p.mirrorAction(m, p.setAdapters))
|
||||
|
||||
mux.HandleFunc("POST /admin/mirror/serve", p.mirrorAction(m,
|
||||
func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
|
||||
@@ -469,6 +480,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
|
||||
Name: m.Name, Mode: modeOf(m.Scaling), Replicas: max(m.Scaling.Max, 1),
|
||||
Min: m.Scaling.Min, Max: m.Scaling.Max,
|
||||
Class: m.Class, EffClass: m.EffectiveClass(), Owner: m.Owner,
|
||||
Adapters: m.Adapters,
|
||||
Declared: p.catalog.Declared(m.Name),
|
||||
}
|
||||
if len(m.Profiles) > 0 {
|
||||
@@ -487,6 +499,8 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
|
||||
}
|
||||
}
|
||||
|
||||
p.heldKinds(r.Context(), d.Rows)
|
||||
|
||||
// The one row whose card is open. From the form too, so a change made in
|
||||
// the card comes back with the card still open to show it.
|
||||
open := r.FormValue("open")
|
||||
@@ -494,6 +508,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
|
||||
d.Rows[i].Open = openKey(d.Rows[i].Model, d.Rows[i].Version)
|
||||
if open != "" && d.Rows[i].Open == open && d.Rows[i].State == mirror.Present {
|
||||
d.Rows[i].Detail = p.heldDetailFor(r.Context(), d.Rows[i])
|
||||
d.Rows[i].Detail.addAdapterChoices(d.Rows[i], d.Rows)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -755,6 +770,46 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string
|
||||
}
|
||||
d.RepoSets = append(d.RepoSets, rs)
|
||||
}
|
||||
p.probeKinds(r.Context(), d.RepoSets)
|
||||
}
|
||||
|
||||
// kindOf reads what a GGUF file is; a variable so tests need no registry.
|
||||
var kindOf = func(ctx context.Context, url string) (gguf.Header, error) {
|
||||
return gguf.KindFromURL(ctx, nil, url)
|
||||
}
|
||||
|
||||
// probeKinds reads the start of each single-file GGUF set, to tell LoRA
|
||||
// adapters from models: a repository tagged "lora" holds a model with the
|
||||
// adapter merged in as often as an adapter, and only the file says which.
|
||||
// A few at once and a few in all, since each is a request to the registry.
|
||||
func (p *AccountPages) probeKinds(ctx context.Context, sets []repoSet) {
|
||||
const most, together = 12, 6
|
||||
sem := make(chan struct{}, together)
|
||||
var wg sync.WaitGroup
|
||||
n := 0
|
||||
for i := range sets {
|
||||
s := &sets[i]
|
||||
if len(s.Sources) != 1 || !gguf.IsGGUF(s.Sources[0].Path) || n == most {
|
||||
continue
|
||||
}
|
||||
n++
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if h, err := kindOf(cctx, s.Sources[0].URL); err == nil {
|
||||
s.Arch = h.Architecture
|
||||
s.Kind = "model"
|
||||
if h.IsAdapter() {
|
||||
s.Kind = "adapter"
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// defaultModelName is the catalog name a repository suggests.
|
||||
|
||||
@@ -27,6 +27,7 @@ type CatalogAdmin interface {
|
||||
SetClass(ctx context.Context, name, class, by string) (catalog.Model, error)
|
||||
SetOwner(ctx context.Context, name, owner, by string) (catalog.Model, error)
|
||||
Resources(ctx context.Context, name, profile string, memoryGB, cpuCores int, by string) (catalog.Model, error)
|
||||
SetAdapters(ctx context.Context, name string, adapters []catalog.Adapter, by string) (catalog.Model, error)
|
||||
}
|
||||
|
||||
// serveForm is the offer to put one mirrored version into service.
|
||||
|
||||
@@ -16,15 +16,16 @@ import (
|
||||
)
|
||||
|
||||
type fakeCatalog struct {
|
||||
added []catalog.Model
|
||||
declared string
|
||||
sources map[string][]catalog.Model
|
||||
scaled []string
|
||||
lastScale catalog.Scaling
|
||||
err error
|
||||
classes []string
|
||||
owners []string
|
||||
resources []string
|
||||
added []catalog.Model
|
||||
declared string
|
||||
sources map[string][]catalog.Model
|
||||
scaled []string
|
||||
lastScale catalog.Scaling
|
||||
err error
|
||||
classes []string
|
||||
owners []string
|
||||
resources []string
|
||||
adapterSets [][]catalog.Adapter
|
||||
}
|
||||
|
||||
func (f *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
|
||||
@@ -39,6 +40,11 @@ func (f *fakeCatalog) Sources(context.Context) (map[string][]catalog.Model, erro
|
||||
return f.sources, nil
|
||||
}
|
||||
|
||||
func (f *fakeCatalog) SetAdapters(_ context.Context, name string, as []catalog.Adapter, _ string) (catalog.Model, error) {
|
||||
f.adapterSets = append(f.adapterSets, as)
|
||||
return catalog.Model{Name: name, Adapters: as}, f.err
|
||||
}
|
||||
|
||||
func (f *fakeCatalog) SetClass(_ context.Context, name, class, _ string) (catalog.Model, error) {
|
||||
f.classes = append(f.classes, name+"="+class)
|
||||
return catalog.Model{Name: name, Class: class}, f.err
|
||||
|
||||
@@ -4,13 +4,19 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
@@ -37,9 +43,15 @@ type heldDetail struct {
|
||||
// heldModel is one catalog model running a held version.
|
||||
type heldModel struct {
|
||||
modelUse
|
||||
View adminapi.ModelView
|
||||
Replicas []heldReplica
|
||||
Headroom *adminapi.HeadroomView
|
||||
// Adapters are the model's own; AdapterChoices the held adapter files
|
||||
// of its architecture not attached yet; FindAdapters the explorer's
|
||||
// list of adapters made for it.
|
||||
Adapters []catalog.Adapter
|
||||
AdapterChoices []adapterChoice
|
||||
FindAdapters template.URL
|
||||
View adminapi.ModelView
|
||||
Replicas []heldReplica
|
||||
Headroom *adminapi.HeadroomView
|
||||
// MostNow is how many replicas could always run, Current plus the room
|
||||
// found; zero when the room was not measured or had no end in sight.
|
||||
MostNow int
|
||||
@@ -74,7 +86,7 @@ func (p *AccountPages) heldDetailFor(ctx context.Context, row mirrorRow) *heldDe
|
||||
}
|
||||
if p.fleet == nil {
|
||||
for _, u := range row.UsedBy {
|
||||
d.Models = append(d.Models, heldModel{modelUse: u})
|
||||
d.Models = append(d.Models, heldModel{modelUse: u, Adapters: u.Adapters})
|
||||
}
|
||||
return d
|
||||
}
|
||||
@@ -88,7 +100,7 @@ func (p *AccountPages) heldDetailFor(ctx context.Context, row mirrorRow) *heldDe
|
||||
}
|
||||
hs, _ := p.fleet.(HeadroomSource)
|
||||
for _, u := range row.UsedBy {
|
||||
hm := heldModel{modelUse: u, View: views[u.Name]}
|
||||
hm := heldModel{modelUse: u, View: views[u.Name], Adapters: u.Adapters}
|
||||
for _, rep := range p.fleet.Replicas(u.Name) {
|
||||
// Built field by field: a ReplicaView carries the engine's key,
|
||||
// which is for the admin listener and never for a page.
|
||||
@@ -282,3 +294,140 @@ func (p *AccountPages) setModel(ctx context.Context, admin store.User, r *http.R
|
||||
}
|
||||
return name + " " + strings.Join(done, "; "), nil
|
||||
}
|
||||
|
||||
// heldKinds reads what each held single-file GGUF is — an adapter or a model —
|
||||
// once per file, remembered after.
|
||||
func (p *AccountPages) heldKinds(ctx context.Context, rows []mirrorRow) {
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, 6)
|
||||
for i := range rows {
|
||||
row := &rows[i]
|
||||
if row.State != mirror.Present || row.Files != 1 || !gguf.IsGGUF(row.File) {
|
||||
continue
|
||||
}
|
||||
if h, ok := p.kinds.Load(row.URL); ok {
|
||||
row.Kind, row.Arch = kindName(h.(gguf.Header)), h.(gguf.Header).Architecture
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
cctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
if h, err := kindOf(cctx, row.URL); err == nil {
|
||||
p.kinds.Store(row.URL, h)
|
||||
row.Kind, row.Arch = kindName(h), h.Architecture
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func kindName(h gguf.Header) string {
|
||||
if h.IsAdapter() {
|
||||
return "adapter"
|
||||
}
|
||||
return "model"
|
||||
}
|
||||
|
||||
// adapterChoice is a held adapter file that could be attached to a model.
|
||||
type adapterChoice struct {
|
||||
URL, Label, Suggest string
|
||||
}
|
||||
|
||||
// addAdapterChoices offers, on each model of a held model's card, the held
|
||||
// adapter files of the same architecture, and a way to find more.
|
||||
func (d *heldDetail) addAdapterChoices(row mirrorRow, rows []mirrorRow) {
|
||||
for i := range d.Models {
|
||||
hm := &d.Models[i]
|
||||
hm.FindAdapters = findAdaptersHref(row.Source)
|
||||
if row.Kind == "adapter" {
|
||||
continue
|
||||
}
|
||||
for _, r := range rows {
|
||||
if r.Kind != "adapter" || (row.Arch != "" && r.Arch != row.Arch) {
|
||||
continue
|
||||
}
|
||||
attached := false
|
||||
for _, a := range hm.Adapters {
|
||||
attached = attached || a.Source == r.URL
|
||||
}
|
||||
if !attached {
|
||||
hm.AdapterChoices = append(hm.AdapterChoices, adapterChoice{URL: r.URL,
|
||||
Label: r.Model + " " + r.Version + " · " + r.File, Suggest: hm.Name + "-" + r.Model})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// findAdaptersHref is the explorer's list of adapters for a held model: those
|
||||
// made for the model its repository was made from, when it came from the
|
||||
// registry, and every adapter otherwise.
|
||||
func findAdaptersHref(source string) template.URL {
|
||||
const base = "/admin/mirror?tab=explore&view=adapters"
|
||||
u, err := url.Parse(source)
|
||||
if err != nil || u.Host != "huggingface.co" {
|
||||
return base
|
||||
}
|
||||
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
|
||||
if len(parts) < 2 {
|
||||
return base
|
||||
}
|
||||
return template.URL(base + "&from=" + url.QueryEscape(parts[0]+"/"+parts[1]))
|
||||
}
|
||||
|
||||
// setAdapters attaches or detaches one adapter on a model, from its card.
|
||||
// An adapter is checked before it is attached: a file that is not an adapter,
|
||||
// or one made for another architecture, is a replica that fails to start.
|
||||
func (p *AccountPages) setAdapters(ctx context.Context, admin store.User, r *http.Request) (string, error) {
|
||||
name := r.FormValue("model")
|
||||
current, err := p.currentModel(ctx, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
adapters := slices.Clone(current.Adapters)
|
||||
switch r.FormValue("op") {
|
||||
case "remove":
|
||||
gone := r.FormValue("name")
|
||||
adapters = slices.DeleteFunc(adapters, func(a catalog.Adapter) bool { return a.Name == gone })
|
||||
if len(adapters) == len(current.Adapters) {
|
||||
return "", fmt.Errorf("%s has no adapter %q", name, gone)
|
||||
}
|
||||
if _, err := p.catalog.SetAdapters(ctx, name, adapters, admin.Email); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s: adapter %s removed; replicas drop it when restarted (llmgw ctl restart --model %s)",
|
||||
name, gone, name), nil
|
||||
case "add":
|
||||
an := strings.TrimSpace(r.FormValue("name"))
|
||||
src := strings.TrimSpace(r.FormValue("source"))
|
||||
scale := 1.0
|
||||
if v := strings.TrimSpace(r.FormValue("scale")); v != "" {
|
||||
if scale, err = strconv.ParseFloat(v, 64); err != nil || scale <= 0 {
|
||||
return "", errors.New("the scale is a positive number")
|
||||
}
|
||||
}
|
||||
if an == "" || src == "" {
|
||||
return "", errors.New("an adapter needs a name and a file")
|
||||
}
|
||||
h, err := kindOf(ctx, src)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the adapter file could not be read: %w", err)
|
||||
}
|
||||
if !h.IsAdapter() {
|
||||
return "", errors.New("that file is a model, not a LoRA adapter")
|
||||
}
|
||||
if base, err := kindOf(ctx, current.Source); err == nil && base.Architecture != h.Architecture {
|
||||
return "", fmt.Errorf("that adapter is for %s models and %s is %s", h.Architecture, name, base.Architecture)
|
||||
}
|
||||
adapters = append(adapters, catalog.Adapter{Name: an, Source: src, Scale: scale})
|
||||
if _, err := p.catalog.SetAdapters(ctx, name, adapters, admin.Email); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s: adapter %s added; ask for it by that name once the replicas are restarted "+
|
||||
"(llmgw ctl restart --model %s)", name, an, name), nil
|
||||
}
|
||||
return "", errors.New("add or remove an adapter")
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -9,6 +10,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
@@ -269,3 +271,65 @@ func TestRoomAHigherClassWouldMakeIsOfferedAndSaid(t *testing.T) {
|
||||
t.Errorf("three is past even what preemption makes: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdapterHeldInTheMirrorIsAttachedFromItsModelsCard(t *testing.T) {
|
||||
prev := kindOf
|
||||
t.Cleanup(func() { kindOf = prev })
|
||||
kindOf = func(_ context.Context, u string) (gguf.Header, error) {
|
||||
switch {
|
||||
case strings.Contains(u, "/sql/"):
|
||||
return gguf.Header{Architecture: "qwen2", Type: "adapter"}, nil
|
||||
case strings.Contains(u, "/llama-lora/"):
|
||||
return gguf.Header{Architecture: "llama", Type: "adapter"}, nil
|
||||
}
|
||||
return gguf.Header{Architecture: "qwen2"}, nil
|
||||
}
|
||||
cat := coderCatalog()
|
||||
coder := &cat.sources["http://mirror:8081/coder/v1/c.gguf"][0]
|
||||
coder.Source = "http://mirror:8081/coder/v1/c.gguf"
|
||||
coder.Profiles = []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 12}}
|
||||
m := &fakeMirror{models: []mirror.Status{
|
||||
{Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 1,
|
||||
Source: "https://huggingface.co/bartowski/Qwen2.5-Coder-7B-Instruct-GGUF/resolve/abc/c.gguf"},
|
||||
{Model: "sql", Version: "v1", File: "sql.gguf", State: mirror.Present, BytesDone: 1},
|
||||
{Model: "llama-lora", Version: "v1", File: "l.gguf", State: mirror.Present, BytesDone: 1},
|
||||
}}
|
||||
mux := mirrorPageDeps(t, MirrorDeps{Mirror: m, Catalog: cat, Fleet: &roomyFleet{more: 1}}, true)
|
||||
|
||||
list := mirrorGet(t, mux, "/admin/mirror").Body.String()
|
||||
if strings.Count(list, ">LoRA adapter</span>") != 2 {
|
||||
t.Error("the two adapter files are not marked as adapters in the list")
|
||||
}
|
||||
card := mirrorGet(t, mux, "/admin/mirror?open=coder:v1").Body.String()
|
||||
if !strings.Contains(card, `<option value="http://mirror:8081/sql/v1/sql.gguf" data-suggest="coder-sql">`) {
|
||||
t.Error("the qwen2 adapter is not offered for the qwen2 model")
|
||||
}
|
||||
if strings.Contains(card, "llama-lora") && strings.Contains(card, `value="http://mirror:8081/llama-lora/v1/l.gguf"`) {
|
||||
t.Error("an adapter for another architecture is offered")
|
||||
}
|
||||
if !strings.Contains(card, "view=adapters&from=bartowski%2FQwen2.5-Coder-7B-Instruct-GGUF") {
|
||||
t.Error("finding adapters should start from the model's own repository")
|
||||
}
|
||||
|
||||
post := func(v url.Values) string {
|
||||
v.Set("csrf", "tok")
|
||||
v.Set("model", "coder")
|
||||
v.Set("open", "coder:v1")
|
||||
return mirrorPost(t, mux, "/admin/mirror/adapters", v).Body.String()
|
||||
}
|
||||
body := post(url.Values{"op": {"add"}, "name": {"coder-sql"}, "source": {"http://mirror:8081/sql/v1/sql.gguf"}, "scale": {"0.8"}})
|
||||
if len(cat.adapterSets) != 1 || len(cat.adapterSets[0]) != 1 || cat.adapterSets[0][0].Scale != 0.8 {
|
||||
t.Fatalf("attached %+v: %s", cat.adapterSets, body)
|
||||
}
|
||||
if !strings.Contains(body, "adapter coder-sql added") {
|
||||
t.Errorf("the attach was not said: %s", body)
|
||||
}
|
||||
for name, src := range map[string]string{"another architecture": "http://mirror:8081/llama-lora/v1/l.gguf",
|
||||
"a model": "http://mirror:8081/coder/v1/c.gguf"} {
|
||||
n := len(cat.adapterSets)
|
||||
out := post(url.Values{"op": {"add"}, "name": {"x"}, "source": {src}})
|
||||
if len(cat.adapterSets) != n || !strings.Contains(out, `class="error"`) {
|
||||
t.Errorf("%s was attached", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,9 @@ const (
|
||||
viewTrending = "trending"
|
||||
viewNew = "new"
|
||||
viewDownloads = "downloads"
|
||||
// viewAdapters lists LoRA adapters, all of them or those made for one
|
||||
// model.
|
||||
viewAdapters = "adapters"
|
||||
)
|
||||
|
||||
// newWithin is how recent a repository has to be to count as new.
|
||||
@@ -31,6 +34,7 @@ const exploreShown = 25
|
||||
|
||||
var exploreViews = []struct{ key, label string }{
|
||||
{viewTrending, "Trending"}, {viewNew, "New this month"}, {viewDownloads, "Most downloaded"},
|
||||
{viewAdapters, "LoRA adapters"},
|
||||
}
|
||||
|
||||
var exploreTasks = []struct{ key, label, tag string }{
|
||||
@@ -48,6 +52,9 @@ var exploreSizes = []struct {
|
||||
// open: a listing, its filters, and links that keep them.
|
||||
type explorer struct {
|
||||
View, Task, Size string
|
||||
// Of is the model whose adapters are listed, in the adapters view; empty
|
||||
// lists adapters for any model.
|
||||
Of string
|
||||
// Here keeps to what the nodes' llama.cpp can load. On by default: a
|
||||
// trending list is mostly things this cluster cannot run, and finding
|
||||
// that out one repository at a time is what the explorer is for avoiding.
|
||||
@@ -123,11 +130,12 @@ func trimZero(s string) string { return strings.TrimSuffix(s, ".0") }
|
||||
// — the page opened from the menu — it is trending chat models this cluster
|
||||
// can run.
|
||||
func exploreState(q url.Values) explorer {
|
||||
e := explorer{View: q.Get("view"), Task: q.Get("task"), Size: q.Get("size")}
|
||||
e := explorer{View: q.Get("view"), Task: q.Get("task"), Size: q.Get("size"),
|
||||
Of: strings.Trim(strings.TrimSpace(q.Get("of")), "/")}
|
||||
// "x" marks a submitted form, so an unticked box reads as off rather than
|
||||
// as not yet chosen.
|
||||
e.Here = q.Get("x") != "1" || q.Get("here") == "1"
|
||||
if !oneOf(e.View, viewTrending, viewNew, viewDownloads) {
|
||||
if !oneOf(e.View, viewTrending, viewNew, viewDownloads, viewAdapters) {
|
||||
e.View = viewTrending
|
||||
// A search with no view asks for what a name usually means, which is
|
||||
// the one everybody downloads.
|
||||
@@ -165,6 +173,10 @@ func (e explorer) href(query, key, value string) template.URL {
|
||||
if key != "" {
|
||||
v.Set(key, value)
|
||||
}
|
||||
// Whose adapters, kept while it is adapters being looked at.
|
||||
if e.Of != "" && v.Get("view") == viewAdapters {
|
||||
v.Set("of", e.Of)
|
||||
}
|
||||
return template.URL("/admin/mirror?" + v.Encode())
|
||||
}
|
||||
|
||||
@@ -181,6 +193,16 @@ func (e explorer) hfQuery(search string) hf.Query {
|
||||
// Trending, then kept to the recent: sorted by creation date alone,
|
||||
// the newest repositories are mostly empty uploads nobody has used.
|
||||
q.Limit = 100
|
||||
case viewAdapters:
|
||||
// Adapters rarely carry a task or a parameter count, so neither is
|
||||
// asked for; what makes one useful is the model it was made for.
|
||||
q.Sort = hf.SortDownloads
|
||||
if e.Of != "" {
|
||||
q.Filters = []string{"base_model:adapter:" + e.Of}
|
||||
} else {
|
||||
q.Filters = []string{"lora"}
|
||||
}
|
||||
return q
|
||||
}
|
||||
for _, t := range exploreTasks {
|
||||
if t.key == e.Task {
|
||||
@@ -198,6 +220,15 @@ func (e explorer) hfQuery(search string) hf.Query {
|
||||
// exploreList fills in the listing.
|
||||
func (p *AccountPages) exploreList(r *http.Request, d *adminMirrorData) {
|
||||
e := exploreState(r.URL.Query())
|
||||
// From a held model's card: its repository, whose adapters are those of
|
||||
// the model it was made from — a GGUF repository is a quantisation, and
|
||||
// adapters are made for the original.
|
||||
if from := strings.TrimSpace(r.URL.Query().Get("from")); e.View == viewAdapters && e.Of == "" && from != "" {
|
||||
e.Of = from
|
||||
if repo, err := p.registry.Repo(r.Context(), from); err == nil && len(repo.BaseModel) > 0 {
|
||||
e.Of = repo.BaseModel[0]
|
||||
}
|
||||
}
|
||||
for _, v := range exploreViews {
|
||||
e.Views = append(e.Views, exploreLink{v.label, e.href(d.Query, "view", v.key), v.key == e.View})
|
||||
}
|
||||
@@ -225,6 +256,14 @@ func (p *AccountPages) exploreList(r *http.Request, d *adminMirrorData) {
|
||||
if a := m.Architecture(); a != "" && p.llamaVersion != "" {
|
||||
row.Verdict = gguf.LlamaCpp(p.llamaVersion, a)
|
||||
}
|
||||
// Adapters are small and say little about their architecture in a
|
||||
// listing; whether one fits is read from its file when it is opened.
|
||||
if e.View == viewAdapters {
|
||||
if len(e.Rows) < exploreShown {
|
||||
e.Rows = append(e.Rows, row)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// An adapter or a fragment, not a model: tens of megabytes of LoRA
|
||||
// weights listed among the models they need a model to run on.
|
||||
if e.Here && m.Params() > 0 && m.Params() < minParams {
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"html"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
)
|
||||
|
||||
@@ -195,3 +198,52 @@ func TestAFetchFromTheListAnswersWithItsOutcomeAlone(t *testing.T) {
|
||||
t.Errorf("a refusal: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// noProbe answers every file as an unreadable one, for tests that are not
|
||||
// about what the files are.
|
||||
func TestMain(m *testing.M) {
|
||||
kindOf = func(context.Context, string) (gguf.Header, error) {
|
||||
return gguf.Header{}, errors.New("not probed in tests")
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func TestAdaptersAreListedForAModelAndToldFromModelsByTheirFiles(t *testing.T) {
|
||||
prev := kindOf
|
||||
t.Cleanup(func() { kindOf = prev })
|
||||
kindOf = func(_ context.Context, url string) (gguf.Header, error) {
|
||||
if strings.Contains(url, "adapter") {
|
||||
return gguf.Header{Architecture: "qwen2", Type: "adapter"}, nil
|
||||
}
|
||||
return gguf.Header{Architecture: "qwen2"}, nil
|
||||
}
|
||||
reg := &fakeRegistry{
|
||||
results: []hf.Model{{ID: "ggml-org/LoRA-Qwen2.5-Coder-7B-Instruct-F16-GGUF", Downloads: 900}},
|
||||
repo: hf.Repo{ID: "ggml-org/LoRA-Qwen2.5-Coder-7B-Instruct-F16-GGUF", Revision: "776530df0000",
|
||||
BaseModel: []string{"Qwen/Qwen2.5-Coder-7B-Instruct"},
|
||||
Files: []hf.File{{Path: "lora-adapter-f16.gguf", Bytes: 726 << 20}, {Path: "merged-Q4_K_M.gguf", Bytes: 4 << 30}}},
|
||||
}
|
||||
mux := mirrorPageWith(t, &fakeMirror{}, reg, true)
|
||||
|
||||
// From a held model's card: its repository is looked up, and the
|
||||
// adapters listed are those of the model it was made from.
|
||||
body := mirrorGet(t, mux, "/admin/mirror?tab=explore&view=adapters&from=bartowski/Qwen2.5-Coder-7B-Instruct-GGUF").Body.String()
|
||||
want := "base_model:adapter:Qwen/Qwen2.5-Coder-7B-Instruct"
|
||||
if len(reg.query.Filters) != 1 || reg.query.Filters[0] != want {
|
||||
t.Errorf("asked the registry for %v, expected %s", reg.query.Filters, want)
|
||||
}
|
||||
if !strings.Contains(body, "LoRA adapters made for <code>Qwen/Qwen2.5-Coder-7B-Instruct</code>") {
|
||||
t.Error("the page does not say whose adapters these are")
|
||||
}
|
||||
if strings.Contains(body, `aria-label="Task"`) {
|
||||
t.Error("task and size filters shown for adapters, which rarely carry either")
|
||||
}
|
||||
|
||||
repo := mirrorGet(t, mux, "/admin/mirror?repo=ggml-org/LoRA-Qwen2.5-Coder-7B-Instruct-F16-GGUF").Body.String()
|
||||
if !strings.Contains(repo, "LoRA adapter · qwen2") {
|
||||
t.Error("the adapter file is not marked as one")
|
||||
}
|
||||
if !strings.Contains(repo, `<span class="opt">model · qwen2</span>`) {
|
||||
t.Error("the merged file is not marked as a model")
|
||||
}
|
||||
}
|
||||
@@ -858,3 +858,7 @@ table.rows td.warn { color: var(--warn); cursor: help; }
|
||||
.held-card .replicas-form select, .capacity-form select { display: block; margin-top: .25rem; }
|
||||
.held-card .replicas-form .field-row { flex-wrap: wrap; gap: .75rem 1rem; }
|
||||
.quota-tree td.q-act form.inline { margin-left: .75rem; }
|
||||
.held-card .adapters { margin: .75rem 0; }
|
||||
.held-card .adapters h4 { font-size: .9rem; margin: 0 0 .35rem; display: flex; gap: .75rem; align-items: baseline; }
|
||||
.held-card .adapter-form .field-row { align-items: flex-end; flex-wrap: wrap; gap: .5rem 1rem; }
|
||||
.held-card .adapter-form select { display: block; margin-top: .25rem; max-width: 28rem; }
|
||||
@@ -47,7 +47,8 @@ reachable from this machine.</p>
|
||||
{{range .Rows}}{{$row0 := .}}<tr{{if $grouped}} class="model-version"{{end}}{{if .Detail}} class="open"{{end}}>
|
||||
<td>{{if eq (printf "%s" .State) "present"}}<a class="open-held" aria-expanded="{{if .Detail}}true{{else}}false{{end}}"
|
||||
href="/admin/mirror?tab=held{{if not .Detail}}&open={{.Open}}{{end}}"><strong>{{.Model}}</strong></a>
|
||||
{{else}}<strong>{{.Model}}</strong>{{end}} <span class="opt">{{.Version}}</span></td>
|
||||
{{else}}<strong>{{.Model}}</strong>{{end}} <span class="opt">{{.Version}}</span>
|
||||
{{if eq .Kind "adapter"}}<span class="tag new" title="a LoRA adapter for {{.Arch}} models: attach it on a model's card">LoRA adapter</span>{{end}}</td>
|
||||
<td class="file" title="{{.File}}{{if .SHA256}} · sha256 {{.SHA256}}{{end}}">
|
||||
{{if gt .Files 1}}{{.Files}} files{{else}}<code>{{.File}}</code>{{end}}</td>
|
||||
<td class="num">
|
||||
@@ -215,16 +216,25 @@ document.addEventListener('click', function (e) {
|
||||
{{if and (eq .Tab "explore") .HasRegistry}}
|
||||
{{if not .Repo}}{{with .Explore}}
|
||||
<nav class="pills" aria-label="Which models">{{range .Views}}<a href="{{.Href}}"{{if .Active}} class="on" aria-current="page"{{end}}>{{.Label}}</a>{{end}}</nav>
|
||||
{{if eq .View "adapters"}}
|
||||
<p class="note">{{if .Of}}LoRA adapters made for <code>{{.Of}}</code> ·
|
||||
<a href="/admin/mirror?tab=explore&view=adapters">all adapters</a>{{else}}LoRA adapters for any model.
|
||||
From a held model's card, <em>find adapters</em> lists those made for it.{{end}}
|
||||
An adapter runs on the model it was made for, and only as a GGUF adapter file;
|
||||
open one to see which of its files are.</p>
|
||||
{{else}}
|
||||
<div class="explore-filters">
|
||||
<nav class="pills small" aria-label="Task">{{range .Tasks}}<a href="{{.Href}}"{{if .Active}} class="on" aria-current="true"{{end}}>{{.Label}}</a>{{end}}</nav>
|
||||
<nav class="pills small" aria-label="Size">{{range .Sizes}}<a href="{{.Href}}"{{if .Active}} class="on" aria-current="true"{{end}}>{{.Label}}</a>{{end}}</nav>
|
||||
</div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
<form method="get" action="/admin/mirror" class="explore-search">
|
||||
<input type="hidden" name="tab" value="explore">
|
||||
{{with .Explore}}<input type="hidden" name="view" value="{{.View}}">
|
||||
<input type="hidden" name="task" value="{{.Task}}">
|
||||
<input type="hidden" name="size" value="{{.Size}}">
|
||||
{{if .Of}}<input type="hidden" name="of" value="{{.Of}}">{{end}}
|
||||
<input type="hidden" name="x" value="1">{{end}}
|
||||
<input type="search" name="q" value="{{.Query}}" placeholder="Search Hugging Face — qwen coder, gemma, bge…" aria-label="Search Hugging Face">
|
||||
<label class="check"><input type="checkbox" name="here" value="1" onchange="this.form.submit()"{{if .Explore}}{{if .Explore.Here}} checked{{end}}{{end}}> only what these nodes can run</label>
|
||||
@@ -405,7 +415,9 @@ version and the old weights stay where they are.</p>
|
||||
<tr><th>What</th><th class="num">Size</th><th>Published checksum</th><th></th></tr>
|
||||
{{range .RepoSets}}<tr>
|
||||
<td><code>{{.Name}}</code>
|
||||
{{if .Shards}}<span class="opt">{{.Shards}} shards, fetched together</span>{{end}}</td>
|
||||
{{if .Shards}}<span class="opt">{{.Shards}} shards, fetched together</span>{{end}}
|
||||
{{if eq .Kind "adapter"}}<span class="tag new" title="a LoRA adapter: it runs on a {{.Arch}} model it was made for, not on its own">LoRA adapter · {{.Arch}}</span>
|
||||
{{else if eq .Kind "model"}}<span class="opt">model{{with .Arch}} · {{.}}{{end}}</span>{{end}}</td>
|
||||
<td class="num">{{bytes .Bytes}}</td>
|
||||
<td>{{with .SHA256}}<code title="{{.}}">{{slice . 0 12}}</code>{{else}}—{{end}}</td>
|
||||
<td>
|
||||
@@ -489,6 +501,43 @@ computes once it has it.</p>
|
||||
<ul>{{range .Blocked}}<li>{{.}}</li>{{end}}</ul></details>{{end}}
|
||||
{{end}}
|
||||
|
||||
{{if not .Declared}}{{$mname := .Name}}
|
||||
<div class="adapters">
|
||||
<h4>LoRA adapters <a class="note" href="{{.FindAdapters}}">find adapters on Hugging Face →</a></h4>
|
||||
{{if .Adapters}}<table class="compact rows">
|
||||
<tr><th>Asked for as</th><th>File</th><th class="num">Scale</th><th></th></tr>
|
||||
{{range .Adapters}}{{$a := .}}<tr>
|
||||
<td><code>{{.Name}}</code></td>
|
||||
<td class="clip" title="{{.Source}}">{{.Source}}</td>
|
||||
<td class="num">{{.EffectiveScale}}</td>
|
||||
<td class="actions"><form method="post" action="/admin/mirror/adapters" class="inline">
|
||||
<input type="hidden" name="csrf" value="{{$csrf}}">
|
||||
<input type="hidden" name="model" value="{{$mname}}">
|
||||
<input type="hidden" name="open" value="{{$row.Open}}">
|
||||
<input type="hidden" name="op" value="remove">
|
||||
<input type="hidden" name="name" value="{{$a.Name}}">
|
||||
<button type="submit" class="link danger">remove</button></form></td>
|
||||
</tr>{{end}}</table>
|
||||
{{else}}<p class="note">None. A request for an adapter's name is served by this model's replicas with it applied.</p>{{end}}
|
||||
{{if .AdapterChoices}}{{$m := .}}
|
||||
<form method="post" action="/admin/mirror/adapters" class="adapter-form">
|
||||
<input type="hidden" name="csrf" value="{{$csrf}}">
|
||||
<input type="hidden" name="model" value="{{.Name}}">
|
||||
<input type="hidden" name="open" value="{{$row.Open}}">
|
||||
<input type="hidden" name="op" value="add">
|
||||
<div class="field-row">
|
||||
<label>Adapter file <select name="source" required
|
||||
onchange="this.form.elements.name.value = this.selectedOptions[0].dataset.suggest">
|
||||
{{range .AdapterChoices}}<option value="{{.URL}}" data-suggest="{{.Suggest}}">{{.Label}}</option>{{end}}
|
||||
</select></label>
|
||||
<label>Asked for as <input type="text" name="name" value="{{(index .AdapterChoices 0).Suggest}}" required></label>
|
||||
<label>Scale <input type="text" name="scale" value="1" size="4"></label>
|
||||
<button type="submit">Attach</button>
|
||||
</div>
|
||||
</form>{{else}}<p class="note">No held adapter file for this model's architecture. Fetch one from Hugging Face first.</p>{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
{{if .Declared}}
|
||||
<p class="note">Set in server.yaml, which is read again on every start: change its replicas there.</p>
|
||||
{{else}}
|
||||
|
||||
Reference in new issue
Block a user