The explorer listed every weight file with its own button. For a repository of alternatives that is right — q4_k_m or q8_0, you want one — and for a sharded model it is fifteen clicks to fetch one thing, with nothing stopping you from losing count and leaving a version with fourteen of its parts. Files are grouped into sets now. A run of "-00002-of-00004" is one row and one button; everything else stays its own row. Not "fetch all the weights", which would be wrong half the time: two safetensors files are halves of one thing and useless apart, two GGUF files are usually alternatives and fetching both spends gigabytes on a copy nobody asked for. The set is the unit that is whole. A repository can hold both the split and the joined copy of the same quantisation — the real Qwen coder repos do — so a shard set keeps the -of- notation in its name rather than collapsing onto the name of the whole file beside it. A set publishes no single checksum. Several files have several, and showing one of them invites comparing it against the wrong thing. The form posts every source in the set, and a failure part way says how many already started: a set that fails on its fourth shard has three arriving, and reading that as "nothing happened" sends somebody to fetch them all again. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ed530133b6
commit
a588646b7c
5 files changed
+277
-31
No files matched your search
@@ -3,6 +3,7 @@ package web
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -36,10 +37,13 @@ type Registry interface {
|
||||
Repo(ctx context.Context, id string) (hf.Repo, error)
|
||||
}
|
||||
|
||||
// repoFile is one file of a browsed repository, ready to be mirrored.
|
||||
type repoFile struct {
|
||||
hf.File
|
||||
Source string
|
||||
// repoSet is one thing worth fetching: a file, or every shard of one.
|
||||
type repoSet struct {
|
||||
hf.FileSet
|
||||
// Sources are every file in the set, in the order the registry listed them.
|
||||
// A shard set goes as one action — a fifteen-shard model is not fifteen
|
||||
// decisions, it is one.
|
||||
Sources []repoSource
|
||||
// Model and Version are what the fetch form starts with. Editable, because
|
||||
// the name a catalog entry uses is the operator's to choose and the
|
||||
// registry's idea of it is only a good guess.
|
||||
@@ -47,6 +51,8 @@ type repoFile struct {
|
||||
Version string
|
||||
}
|
||||
|
||||
type repoSource struct{ URL, Path string }
|
||||
|
||||
// mirrorRow is one model version as the page shows it.
|
||||
type mirrorRow struct {
|
||||
mirror.Status
|
||||
@@ -79,7 +85,7 @@ type adminMirrorData struct {
|
||||
Query string
|
||||
Results []hf.Model
|
||||
Repo *hf.Repo
|
||||
RepoFiles []repoFile
|
||||
RepoSets []repoSet
|
||||
RegistryErr string
|
||||
|
||||
Message string
|
||||
@@ -112,23 +118,39 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, m MirrorAdmin, re
|
||||
|
||||
mux.HandleFunc("POST /admin/mirror/fetch", p.mirrorAction(m,
|
||||
func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
|
||||
req := mirror.FetchRequest{
|
||||
Source: strings.TrimSpace(r.FormValue("source")),
|
||||
Model: strings.TrimSpace(r.FormValue("model")),
|
||||
Version: strings.TrimSpace(r.FormValue("version")),
|
||||
File: strings.TrimSpace(r.FormValue("file")),
|
||||
}
|
||||
if req.Source == "" {
|
||||
// One form posts one source or fifteen. A sharded model is one
|
||||
// decision, not one per shard, and the mirror fetches each file
|
||||
// independently anyway.
|
||||
sources, files := r.Form["source"], r.Form["file"]
|
||||
if len(sources) == 0 || strings.TrimSpace(sources[0]) == "" {
|
||||
return "", errors.New("a source URL is required")
|
||||
}
|
||||
got, err := m.Fetch(ctx, req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
model := strings.TrimSpace(r.FormValue("model"))
|
||||
version := strings.TrimSpace(r.FormValue("version"))
|
||||
|
||||
var started int
|
||||
for i, src := range sources {
|
||||
src = strings.TrimSpace(src)
|
||||
if src == "" {
|
||||
continue
|
||||
}
|
||||
req := mirror.FetchRequest{Source: src, Model: model, Version: version}
|
||||
if i < len(files) {
|
||||
req.File = strings.TrimSpace(files[i])
|
||||
}
|
||||
if _, err := m.Fetch(ctx, req); err != nil {
|
||||
// Reported with what already went, so a set that fails on
|
||||
// its fourth shard does not read as though nothing started.
|
||||
return "", fmt.Errorf("%d of %d started, then %s: %w",
|
||||
started, len(sources), mirror.LastPathElement(src), err)
|
||||
}
|
||||
started++
|
||||
}
|
||||
// Named as started, not as done. The mirror answered 202 and the
|
||||
// bytes are still moving; a message saying "added" would invite
|
||||
// somebody to point the catalog at a partial file.
|
||||
return "started downloading " + got.Status.Model + "/" + got.Status.Version, nil
|
||||
return fmt.Sprintf("started downloading %s/%s (%s)",
|
||||
model, version, plural(started, "file", "files")), nil
|
||||
}))
|
||||
|
||||
mux.HandleFunc("POST /admin/mirror/{model}/{version}/remove", p.mirrorAction(m,
|
||||
@@ -278,6 +300,7 @@ func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) {
|
||||
return
|
||||
}
|
||||
d.Repo = &got
|
||||
var weights []hf.File
|
||||
for _, f := range got.Files {
|
||||
if !f.IsWeights() {
|
||||
// Thirty files of which four are worth gigabytes. The rest are
|
||||
@@ -285,15 +308,22 @@ func (p *AccountPages) explore(r *http.Request, d *adminMirrorData) {
|
||||
// somebody came for.
|
||||
continue
|
||||
}
|
||||
d.RepoFiles = append(d.RepoFiles, repoFile{
|
||||
File: f,
|
||||
Source: got.DownloadURL("", f.Path),
|
||||
Model: defaultModelName(got.ID),
|
||||
weights = append(weights, f)
|
||||
}
|
||||
for _, set := range hf.Group(weights) {
|
||||
rs := repoSet{
|
||||
FileSet: set,
|
||||
Model: defaultModelName(got.ID),
|
||||
// The revision, not the date: two fetches of the same commit
|
||||
// land in the same place, and a repository re-uploaded under
|
||||
// the same branch lands somewhere else.
|
||||
Version: got.ShortRevision(),
|
||||
})
|
||||
}
|
||||
for _, f := range set.Files {
|
||||
rs.Sources = append(rs.Sources,
|
||||
repoSource{URL: got.DownloadURL("", f.Path), Path: f.Path})
|
||||
}
|
||||
d.RepoSets = append(d.RepoSets, rs)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user