Trace weights fetched before origins were kept, and let an origin be given by hand
The mirror keeps where it fetched a file beside it, and reports that and the checksum after it restarts. A model whose files the mirror cannot trace is found on the registry: a repository holding a file of that name that has the commit the mirror's directory is named after, with every file at the same size, the most downloaded first. ctl model origin and the console set one by hand, checked the same way. Commits compare shortened. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
dac7ff0e99
commit
f36c46a881
11 files changed
+356
-14
No files matched your search
@@ -36,7 +36,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
|
||||
newTargetCmd(client, asJSON), newAdaptersCmd(client),
|
||||
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
|
||||
newModelRollbackCmd(client))
|
||||
newModelRollbackCmd(client), newModelOriginCmd(client))
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
@@ -117,3 +117,25 @@ func newModelRollbackCmd(client func() *ctlClient) *cobra.Command {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newModelOriginCmd(client func() *ctlClient) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "origin MODEL OWNER/REPO",
|
||||
Short: "Say which registry repository a model's weights came from",
|
||||
Long: "For a model fetched before llmgw kept this, and that it could not find on\n" +
|
||||
"its own. The repository is checked: it must have the commit the mirror's\n" +
|
||||
"directory is named after, holding the model's files at the sizes the\n" +
|
||||
"mirror has. From then on its updates are watched.",
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
body, _ := json.Marshal(map[string]string{"repo": args[1]})
|
||||
var o struct{ Repo, Revision string }
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/origin", body, &o); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s came from %s at %s; its updates are watched\n", args[0], o.Repo,
|
||||
catalog.ShortRevision(o.Revision))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -680,7 +680,33 @@ func (m *consoleModel) doEntry() tea.Cmd {
|
||||
// updateItems are what can be done about a model's repository publishing new
|
||||
// weights: move to them, ask now, or go back.
|
||||
func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem {
|
||||
origin := menuItem{"Where its weights came from…", func(m *consoleModel) tea.Cmd {
|
||||
repo := ""
|
||||
if md.Origin != nil {
|
||||
repo = md.Origin.Repo
|
||||
}
|
||||
m.openForm(&formDialog{title: "Origin of " + md.Name,
|
||||
note: "The registry repository its files were fetched from. Checked: it must have the\n" +
|
||||
"commit the mirror's directory is named after, with the same files at the same sizes.",
|
||||
fields: []formField{textF("repository", repo, "owner/name, like bartowski/phi-4-GGUF")},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
if !strings.Contains(v[0], "/") {
|
||||
return nil, fmt.Errorf("owner/name")
|
||||
}
|
||||
return m.call("Origin of "+md.Name, http.MethodPost, "/api/v1/models/"+url.PathEscape(md.Name)+"/origin",
|
||||
map[string]string{"repo": v[0]}, func(raw json.RawMessage) string {
|
||||
var o adminapi.OriginView
|
||||
_ = json.Unmarshal(raw, &o)
|
||||
return fmt.Sprintf("%s came from %s at %s; its updates are watched.", md.Name, o.Repo,
|
||||
catalog.ShortRevision(o.Revision))
|
||||
}), nil
|
||||
}})
|
||||
return nil
|
||||
}}
|
||||
if md.Origin == nil {
|
||||
if strings.Contains(md.Source, "://") {
|
||||
return []menuItem{origin}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
server := "--server=" + m.client.base
|
||||
@@ -712,6 +738,7 @@ func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem {
|
||||
return fmt.Sprintf("%s at %s: %s", u.Repo, catalog.ShortRevision(u.Latest), updateCell(u))
|
||||
})
|
||||
}})
|
||||
items = append(items, origin)
|
||||
if prev := md.Origin.Previous; prev != "" {
|
||||
items = append(items, menuItem{"Go back to " + catalog.ShortRevision(prev) + "…", func(m *consoleModel) tea.Cmd {
|
||||
m.confirmD = &confirmDialog{title: "Go back on " + md.Name,
|
||||
|
||||
@@ -125,6 +125,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/updates/check", a.checkUpdates)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/origin", a.setOrigin)
|
||||
mux.HandleFunc("GET /api/v1/requests/{id}", a.request)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
|
||||
mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -1520,6 +1521,29 @@ func (a *API) rollbackModel(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]string{"result": msg})
|
||||
}
|
||||
|
||||
// OriginRequest names the repository a model's weights came from.
|
||||
type OriginRequest struct {
|
||||
Repo string `json:"repo"`
|
||||
}
|
||||
|
||||
func (a *API) setOrigin(w http.ResponseWriter, r *http.Request) {
|
||||
if a.serve == nil {
|
||||
writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one")
|
||||
return
|
||||
}
|
||||
var req OriginRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil || req.Repo == "" {
|
||||
writeError(w, http.StatusBadRequest, "name the repository, owner/name")
|
||||
return
|
||||
}
|
||||
o, err := a.serve.SetOriginOf(r.Context(), r.PathValue("name"), req.Repo)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, OriginView{Repo: o.Repo, Revision: o.Revision})
|
||||
}
|
||||
|
||||
// actorOf is who a change is recorded as made by: what the caller says, since
|
||||
// the admin API answers only this machine and trusts whoever reaches it. Kept
|
||||
// to a short printable name, so a claim cannot pass for more than one — no
|
||||
|
||||
@@ -146,6 +146,9 @@ type Server struct {
|
||||
seq int
|
||||
// updates is what the last checks of the registry found, by model.
|
||||
updates map[string]UpdateInfo
|
||||
// origins are those worked out for models whose definition does not
|
||||
// keep one, nil for a model that could not be traced.
|
||||
origins map[string]*catalog.Origin
|
||||
}
|
||||
|
||||
// New builds a server.
|
||||
@@ -162,7 +165,7 @@ func New(d Deps) *Server {
|
||||
if d.Log == nil {
|
||||
d.Log = slog.Default()
|
||||
}
|
||||
return &Server{deps: d, jobs: map[string]*Job{}, updates: map[string]UpdateInfo{}}
|
||||
return &Server{deps: d, jobs: map[string]*Job{}, updates: map[string]UpdateInfo{}, origins: map[string]*catalog.Origin{}}
|
||||
}
|
||||
|
||||
var repoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
@@ -124,35 +125,156 @@ func originFiles(set hf.FileSet) []catalog.OriginFile {
|
||||
return out
|
||||
}
|
||||
|
||||
// originOf is a model's origin: recorded, or worked out from the mirror and
|
||||
// then recorded.
|
||||
// Finder is a registry that can be searched, and read at a past commit: what
|
||||
// finding the origin of weights fetched before origins were kept needs.
|
||||
type Finder interface {
|
||||
Explore(ctx context.Context, q hf.Query) ([]hf.Model, error)
|
||||
RepoAt(ctx context.Context, id, rev string) (hf.Repo, error)
|
||||
}
|
||||
|
||||
// originOf is a model's origin: recorded; or worked out from where the
|
||||
// mirror fetched its files; or found on the registry by the files' names,
|
||||
// the commit the mirror's directory is named after, and their sizes. What is
|
||||
// worked out is recorded, so it is worked out once.
|
||||
func (s *Server) originOf(ctx context.Context, m catalog.Model, list mirror.ModelsResponse) *catalog.Origin {
|
||||
if m.Origin != nil && m.Origin.Repo != "" {
|
||||
return m.Origin
|
||||
}
|
||||
s.mu.Lock()
|
||||
o, known := s.origins[m.Name]
|
||||
s.mu.Unlock()
|
||||
if known {
|
||||
return o
|
||||
}
|
||||
p, ok := placeOf(m.Source, list)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
o, ok := inferOrigin(p)
|
||||
o, ok = inferOrigin(p)
|
||||
if !ok {
|
||||
o = s.findOrigin(ctx, p)
|
||||
}
|
||||
s.remember(ctx, m.Name, o)
|
||||
return o
|
||||
}
|
||||
|
||||
// remember keeps an origin: in the catalog for a model added at runtime, and
|
||||
// here for one from configuration, whose definition is not this one's to
|
||||
// change. A nil origin is remembered too, so a model nobody can trace is not
|
||||
// searched for at every check.
|
||||
func (s *Server) remember(ctx context.Context, name string, o *catalog.Origin) {
|
||||
s.mu.Lock()
|
||||
s.origins[name] = o
|
||||
s.mu.Unlock()
|
||||
if o == nil || s.deps.Models == nil || s.deps.Models.Declared(name) {
|
||||
return
|
||||
}
|
||||
if err := s.deps.Models.SetOrigin(ctx, name, o); err != nil {
|
||||
s.deps.Log.Debug("could not record a model's origin", "model", name, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
var hexRev = regexp.MustCompile(`^[0-9a-f]{7,40}$`)
|
||||
|
||||
// findOrigin looks for the repository a mirror's files came from: one that
|
||||
// holds a file of the same name, and has the commit the mirror's directory
|
||||
// is named after with every file at the same size. Searched by the mirror's
|
||||
// name for the model and the file's own; among the repositories that match,
|
||||
// the most downloaded, which is the original rather than a copy of it.
|
||||
func (s *Server) findOrigin(ctx context.Context, p place) *catalog.Origin {
|
||||
f, ok := s.deps.Registry.(Finder)
|
||||
if !ok || !hexRev.MatchString(p.version) {
|
||||
return nil
|
||||
}
|
||||
if s.deps.Models != nil {
|
||||
if err := s.deps.Models.SetOrigin(ctx, m.Name, o); err != nil {
|
||||
s.deps.Log.Debug("could not record a model's origin", "model", m.Name, "err", err)
|
||||
file := p.files[0].File
|
||||
stem := strings.TrimSuffix(file, ".gguf")
|
||||
if i := strings.LastIndexAny(stem, "-."); i > 0 {
|
||||
stem = stem[:i] // the quantisation
|
||||
}
|
||||
tried := map[string]bool{}
|
||||
for _, q := range []string{p.model, stem} {
|
||||
found, err := f.Explore(ctx, hf.Query{Search: q, GGUF: true, Sort: hf.SortDownloads, Limit: 100, WithFiles: true})
|
||||
if err != nil {
|
||||
s.deps.Log.Debug("searching the registry for a model's origin", "model", p.model, "err", err)
|
||||
continue
|
||||
}
|
||||
for _, c := range found {
|
||||
if tried[c.ID] || !slices.ContainsFunc(c.Siblings, func(sb struct {
|
||||
RFilename string `json:"rfilename"`
|
||||
}) bool {
|
||||
return mirror.LastPathElement(sb.RFilename) == file
|
||||
}) {
|
||||
continue
|
||||
}
|
||||
tried[c.ID] = true
|
||||
if o := s.verifyOrigin(ctx, f, c.ID, p); o != nil {
|
||||
s.deps.Log.Info("found where a model's weights came from", "model", p.model, "repo", c.ID,
|
||||
"revision", catalog.ShortRevision(o.Revision))
|
||||
return o
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifyOrigin reads a repository at the mirror's commit and says whether
|
||||
// every file the mirror holds is there at the same size.
|
||||
func (s *Server) verifyOrigin(ctx context.Context, f Finder, id string, p place) *catalog.Origin {
|
||||
repo, err := f.RepoAt(ctx, id, p.version)
|
||||
if err != nil || !strings.HasPrefix(repo.Revision, p.version) {
|
||||
return nil
|
||||
}
|
||||
o := &catalog.Origin{Repo: repo.ID, Revision: repo.Revision}
|
||||
for _, st := range p.files {
|
||||
i := slices.IndexFunc(repo.Files, func(g hf.File) bool {
|
||||
return mirror.LastPathElement(g.Path) == st.File && g.Bytes == st.BytesDone
|
||||
})
|
||||
if i < 0 {
|
||||
return nil
|
||||
}
|
||||
g := repo.Files[i]
|
||||
o.Files = append(o.Files, catalog.OriginFile{Path: g.Path, SHA256: g.SHA256, Bytes: g.Bytes})
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// SetOriginOf records, by hand, the repository a model's weights came from;
|
||||
// the commit is the one the mirror's directory is named after, and the files
|
||||
// must be there at the sizes the mirror holds.
|
||||
func (s *Server) SetOriginOf(ctx context.Context, name, repoID string) (*catalog.Origin, error) {
|
||||
m, err := s.current(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
list, err := s.deps.Mirror.List(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the mirror: %w", err)
|
||||
}
|
||||
p, ok := placeOf(m.Source, list)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s does not run from the mirror", name)
|
||||
}
|
||||
f, ok := s.deps.Registry.(Finder)
|
||||
if !ok {
|
||||
return nil, errors.New("this registry cannot be read at a past commit")
|
||||
}
|
||||
o := s.verifyOrigin(ctx, f, repoID, p)
|
||||
if o == nil {
|
||||
return nil, fmt.Errorf("%s has no commit %s holding %s at the size the mirror has", repoID, p.version,
|
||||
p.files[0].File)
|
||||
}
|
||||
s.remember(ctx, name, o)
|
||||
s.checkOne(ctx, m, o)
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// compare says what the registry's repository has against an origin.
|
||||
func compare(m catalog.Model, o *catalog.Origin, repo hf.Repo) UpdateInfo {
|
||||
u := UpdateInfo{Model: m.Name, Repo: o.Repo, Current: o.Revision, Latest: repo.Revision, CheckedAt: time.Now()}
|
||||
if o.Previous != nil {
|
||||
u.Previous = o.Previous.Revision
|
||||
}
|
||||
if repo.Revision == o.Revision {
|
||||
if sameRevision(repo.Revision, o.Revision) {
|
||||
return u
|
||||
}
|
||||
for _, f := range o.Files {
|
||||
@@ -172,6 +294,14 @@ func compare(m catalog.Model, o *catalog.Origin, repo hf.Repo) UpdateInfo {
|
||||
return u
|
||||
}
|
||||
|
||||
// sameRevision compares commits either of which may be shortened.
|
||||
func sameRevision(a, b string) bool {
|
||||
if a == "" || b == "" {
|
||||
return a == b
|
||||
}
|
||||
return strings.HasPrefix(a, b) || strings.HasPrefix(b, a)
|
||||
}
|
||||
|
||||
// CheckUpdates asks the registry about every model with an origin.
|
||||
func (s *Server) CheckUpdates(ctx context.Context) {
|
||||
if s.deps.Current == nil {
|
||||
@@ -228,9 +358,16 @@ func (s *Server) CheckModel(ctx context.Context, name string) (UpdateInfo, error
|
||||
if err != nil {
|
||||
return UpdateInfo{}, fmt.Errorf("the mirror: %w", err)
|
||||
}
|
||||
// Asked by hand: one that could not be traced is looked for again.
|
||||
s.mu.Lock()
|
||||
if o, ok := s.origins[name]; ok && o == nil {
|
||||
delete(s.origins, name)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
o := s.originOf(ctx, m, list)
|
||||
if o == nil {
|
||||
return UpdateInfo{}, fmt.Errorf("%s did not come from the registry through the mirror, so there is nothing to compare", name)
|
||||
return UpdateInfo{}, fmt.Errorf("where %s's weights came from is not known: give its repository with "+
|
||||
"`llmgw ctl model origin %s OWNER/REPO`", name, name)
|
||||
}
|
||||
return s.checkOne(ctx, m, o), nil
|
||||
}
|
||||
@@ -337,7 +474,7 @@ func (s *Server) runUpdate(j *Job) {
|
||||
s.mu.Unlock()
|
||||
u := compare(m, o, repo)
|
||||
switch {
|
||||
case repo.Revision == o.Revision:
|
||||
case sameRevision(repo.Revision, o.Revision):
|
||||
s.fail(j, "registry", fmt.Errorf("%s already runs %s's latest commit, %s", name, repo.ID, catalog.ShortRevision(repo.Revision)))
|
||||
return
|
||||
case len(u.Missing) > 0:
|
||||
|
||||
@@ -2,6 +2,7 @@ package modelserve
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -262,3 +263,78 @@ func TestRegistryDownloadURLsAreRead(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// findingRegistry is a registry with history: searchable, and readable at a
|
||||
// past commit.
|
||||
type findingRegistry struct {
|
||||
movingRegistry
|
||||
found []hf.Model
|
||||
at map[string]hf.Repo // id@rev
|
||||
}
|
||||
|
||||
func (r *findingRegistry) Explore(context.Context, hf.Query) ([]hf.Model, error) { return r.found, nil }
|
||||
func (r *findingRegistry) RepoAt(_ context.Context, id, rev string) (hf.Repo, error) {
|
||||
if repo, ok := r.at[id+"@"+rev]; ok {
|
||||
return repo, nil
|
||||
}
|
||||
return hf.Repo{}, errors.New("404")
|
||||
}
|
||||
|
||||
func sibling(names ...string) []struct {
|
||||
RFilename string `json:"rfilename"`
|
||||
} {
|
||||
var out []struct {
|
||||
RFilename string `json:"rfilename"`
|
||||
}
|
||||
for _, n := range names {
|
||||
out = append(out, struct {
|
||||
RFilename string `json:"rfilename"`
|
||||
}{n})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestWeightsFetchedBeforeOriginsWereKeptAreTracedToTheirRepository(t *testing.T) {
|
||||
s, mir, models, reg, _ := updSetup(t)
|
||||
// The mirror kept neither source nor checksum.
|
||||
mir.held[0].Source, mir.held[0].SHA256 = "", ""
|
||||
fr := &findingRegistry{movingRegistry: movingRegistry{repo: reg.repo},
|
||||
found: []hf.Model{
|
||||
{ID: "copycat/r", Siblings: sibling("q4.gguf")}, // the right name, not the commit
|
||||
{ID: "other/thing", Siblings: sibling("q8.gguf")}, // not the file
|
||||
{ID: "o/r", Siblings: sibling("README.md", "q4.gguf")},
|
||||
},
|
||||
at: map[string]hf.Repo{
|
||||
"copycat/r@abc12345": {ID: "copycat/r", Revision: "ffff", Files: []hf.File{{Path: "q4.gguf", Bytes: 1 << 30}}},
|
||||
"o/r@abc12345": {ID: "o/r", Revision: oldRev, Files: []hf.File{
|
||||
{Path: "q4.gguf", SHA256: "aaaa", Bytes: 1 << 30}}},
|
||||
}}
|
||||
s.deps.Registry = fr
|
||||
s.CheckUpdates(context.Background())
|
||||
o := models.models["qwen"].Origin
|
||||
if o == nil || o.Repo != "o/r" || o.Revision != oldRev || o.Files[0].SHA256 != "aaaa" {
|
||||
t.Fatalf("traced to %+v", o)
|
||||
}
|
||||
if u := s.Updates()["qwen"]; !u.Available {
|
||||
t.Errorf("%+v", u)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOriginGivenByHandIsChecked(t *testing.T) {
|
||||
s, mir, models, reg, _ := updSetup(t)
|
||||
mir.held[0].Source = ""
|
||||
fr := &findingRegistry{movingRegistry: movingRegistry{repo: reg.repo}, at: map[string]hf.Repo{
|
||||
"o/r@abc12345": {ID: "o/r", Revision: oldRev, Files: []hf.File{{Path: "q4.gguf", SHA256: "aaaa", Bytes: 1 << 30}}},
|
||||
"bad/r@abc12345": {ID: "bad/r", Revision: oldRev, Files: []hf.File{{Path: "q4.gguf", Bytes: 5}}},
|
||||
}}
|
||||
s.deps.Registry = fr
|
||||
if _, err := s.SetOriginOf(context.Background(), "qwen", "bad/r"); err == nil {
|
||||
t.Error("a repository whose file is another size was taken")
|
||||
}
|
||||
if _, err := s.SetOriginOf(context.Background(), "qwen", "o/r"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if o := models.models["qwen"].Origin; o == nil || o.Repo != "o/r" {
|
||||
t.Fatalf("%+v", o)
|
||||
}
|
||||
}
|
||||
+21
-3
@@ -68,6 +68,10 @@ type Model struct {
|
||||
// GGUF is what the registry read from the repository's weights, absent
|
||||
// for one that has none.
|
||||
GGUF *GGUFInfo `json:"gguf"`
|
||||
// Siblings are the repository's files, when asked for with WithFiles.
|
||||
Siblings []struct {
|
||||
RFilename string `json:"rfilename"`
|
||||
} `json:"siblings,omitempty"`
|
||||
}
|
||||
|
||||
// GGUFInfo is the registry's summary of a repository's GGUF weights.
|
||||
@@ -160,6 +164,9 @@ type Query struct {
|
||||
// "lora", or "base_model:adapter:Qwen/Qwen2.5-7B-Instruct" for the
|
||||
// adapters made for one model.
|
||||
Filters []string
|
||||
// WithFiles lists each repository's files too, to find one holding a
|
||||
// file by name.
|
||||
WithFiles bool
|
||||
}
|
||||
|
||||
// exploreFields is everything a listing shows. Asking for any field with
|
||||
@@ -185,6 +192,9 @@ func (c *Client) Explore(ctx context.Context, q Query) ([]Model, error) {
|
||||
"direction": {"-1"},
|
||||
"expand[]": exploreFields,
|
||||
}
|
||||
if q.WithFiles {
|
||||
v.Add("expand[]", "siblings")
|
||||
}
|
||||
if s := strings.TrimSpace(q.Search); s != "" {
|
||||
v.Set("search", s)
|
||||
}
|
||||
@@ -330,7 +340,11 @@ func (r Repo) ShortRevision() string {
|
||||
}
|
||||
|
||||
// Repo reads a repository's revision and file list.
|
||||
func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
|
||||
func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { return c.RepoAt(ctx, id, "") }
|
||||
|
||||
// RepoAt reads a repository as it was at a commit, which may be shortened;
|
||||
// "" is its latest.
|
||||
func (c *Client) RepoAt(ctx context.Context, id, rev string) (Repo, error) {
|
||||
id = strings.Trim(strings.TrimSpace(id), "/")
|
||||
if !strings.Contains(id, "/") {
|
||||
return Repo{}, fmt.Errorf("%q is not a repository; it looks like owner/name", id)
|
||||
@@ -351,7 +365,11 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
|
||||
}
|
||||
// Expanded to what this reads: asking for gguf alone would leave out the
|
||||
// revision and the gate.
|
||||
if err := c.get(ctx, "/api/models/"+id+"?expand[]=sha&expand[]=gated&expand[]=gguf&expand[]=cardData", &meta); err != nil {
|
||||
at := ""
|
||||
if rev != "" {
|
||||
at = "/revision/" + url.PathEscape(rev)
|
||||
}
|
||||
if err := c.get(ctx, "/api/models/"+id+at+"?expand[]=sha&expand[]=gated&expand[]=gguf&expand[]=cardData", &meta); err != nil {
|
||||
return Repo{}, err
|
||||
}
|
||||
|
||||
@@ -366,7 +384,7 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) {
|
||||
}
|
||||
// By revision rather than by branch, so the listing and the URLs built from
|
||||
// it describe the same commit even if the branch moves in between.
|
||||
rev := meta.SHA
|
||||
rev = meta.SHA
|
||||
if rev == "" {
|
||||
rev = "main"
|
||||
}
|
||||
|
||||
@@ -101,6 +101,10 @@ func addWith(ctx context.Context, client *http.Client, root, source string, e En
|
||||
_ = os.Remove(part)
|
||||
return Added{}, err
|
||||
}
|
||||
if err := writeSource(final, source); err != nil {
|
||||
_ = os.Remove(part)
|
||||
return Added{}, err
|
||||
}
|
||||
if err := os.Rename(part, final); err != nil {
|
||||
return Added{}, err
|
||||
}
|
||||
@@ -156,6 +160,7 @@ func Remove(root string, e Entry) error {
|
||||
// Its checksum goes with it; left, it would keep the directory from
|
||||
// being empty, and so from going.
|
||||
_ = os.Remove(checksumPath(path))
|
||||
_ = os.Remove(sourcePath(path))
|
||||
versionDir := filepath.Dir(path)
|
||||
if empty, _ := isEmpty(versionDir); empty {
|
||||
_ = os.Remove(versionDir)
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
@@ -289,6 +290,10 @@ func (m *Manager) All() ([]Status, error) {
|
||||
// verifiable — the download finishing was what made it disappear.
|
||||
if k, ok := known[key]; ok {
|
||||
st.SHA256, st.Source = k.SHA256, k.Source
|
||||
} else {
|
||||
// Fetched before this process started: what was kept beside it.
|
||||
path := filepath.Join(m.root, e.Model, e.Version, e.File)
|
||||
st.SHA256, st.Source = readChecksum(path), readSource(path)
|
||||
}
|
||||
out = append(out, st)
|
||||
seen[key] = struct{}{}
|
||||
|
||||
@@ -187,6 +187,31 @@ func writeChecksum(file, sum string) error {
|
||||
return os.Rename(tmp, checksumPath(file))
|
||||
}
|
||||
|
||||
// sourcePath is where the URL a file was fetched from is kept, beside it like
|
||||
// its checksum: what says, after the mirror restarts, which repository and
|
||||
// commit a file is, so a newer one there can be noticed.
|
||||
func sourcePath(file string) string {
|
||||
return filepath.Join(filepath.Dir(file), "."+filepath.Base(file)+".source")
|
||||
}
|
||||
|
||||
func writeSource(file, source string) error {
|
||||
tmp := sourcePath(file) + partSuffix
|
||||
if err := os.WriteFile(tmp, []byte(source+"\n"), 0o644); err != nil { //nolint:gosec // a URL, for anyone to read
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, sourcePath(file))
|
||||
}
|
||||
|
||||
// readSource is where a file was fetched from, or "" for one fetched before
|
||||
// this was kept, or put in place by hand.
|
||||
func readSource(file string) string {
|
||||
b, err := os.ReadFile(sourcePath(file)) //nolint:gosec // beside a file in the mirror
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
|
||||
// readChecksum is a file's recorded sha256, or "" for a file put in place by
|
||||
// hand, which has none.
|
||||
func readChecksum(file string) string {
|
||||
|
||||
Reference in new issue
Block a user