From f36c46a8813d56d7618b600dd03870458dc77547 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 00:24:41 +0200 Subject: [PATCH] Trace weights fetched before origins were kept, and let an origin be given by hand The mirror keeps where it fetched a file beside it, and reports that and the checksum after it restarts. A model whose files the mirror cannot trace is found on the registry: a repository holding a file of that name that has the commit the mirror's directory is named after, with every file at the same size, the most downloaded first. ctl model origin and the console set one by hand, checked the same way. Commits compare shortened. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/cli/model.go | 2 +- internal/cli/model_update.go | 22 +++ internal/cli/tui_actions.go | 27 ++++ internal/control/adminapi/api.go | 24 +++ internal/control/modelserve/modelserve.go | 5 +- internal/control/modelserve/updates.go | 155 ++++++++++++++++++-- internal/control/modelserve/updates_test.go | 76 ++++++++++ internal/hf/hf.go | 24 ++- internal/mirror/add.go | 5 + internal/mirror/manager.go | 5 + internal/mirror/mirror.go | 25 ++++ 11 files changed, 356 insertions(+), 14 deletions(-) diff --git a/internal/cli/model.go b/internal/cli/model.go index 94e4f48..66c6d28 100644 --- a/internal/cli/model.go +++ b/internal/cli/model.go @@ -36,7 +36,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client), newTargetCmd(client, asJSON), newAdaptersCmd(client), newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client), - newModelRollbackCmd(client)) + newModelRollbackCmd(client), newModelOriginCmd(client)) return c } diff --git a/internal/cli/model_update.go b/internal/cli/model_update.go index 4111663..fe985dd 100644 --- a/internal/cli/model_update.go +++ b/internal/cli/model_update.go @@ -117,3 +117,25 @@ func newModelRollbackCmd(client func() *ctlClient) *cobra.Command { }, } } + +func newModelOriginCmd(client func() *ctlClient) *cobra.Command { + return &cobra.Command{ + Use: "origin MODEL OWNER/REPO", + Short: "Say which registry repository a model's weights came from", + Long: "For a model fetched before llmgw kept this, and that it could not find on\n" + + "its own. The repository is checked: it must have the commit the mirror's\n" + + "directory is named after, holding the model's files at the sizes the\n" + + "mirror has. From then on its updates are watched.", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + body, _ := json.Marshal(map[string]string{"repo": args[1]}) + var o struct{ Repo, Revision string } + if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/origin", body, &o); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "%s came from %s at %s; its updates are watched\n", args[0], o.Repo, + catalog.ShortRevision(o.Revision)) + return nil + }, + } +} diff --git a/internal/cli/tui_actions.go b/internal/cli/tui_actions.go index 05a0784..a4743c8 100644 --- a/internal/cli/tui_actions.go +++ b/internal/cli/tui_actions.go @@ -680,7 +680,33 @@ func (m *consoleModel) doEntry() tea.Cmd { // updateItems are what can be done about a model's repository publishing new // weights: move to them, ask now, or go back. func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem { + origin := menuItem{"Where its weights came from…", func(m *consoleModel) tea.Cmd { + repo := "" + if md.Origin != nil { + repo = md.Origin.Repo + } + m.openForm(&formDialog{title: "Origin of " + md.Name, + note: "The registry repository its files were fetched from. Checked: it must have the\n" + + "commit the mirror's directory is named after, with the same files at the same sizes.", + fields: []formField{textF("repository", repo, "owner/name, like bartowski/phi-4-GGUF")}, + submit: func(m *consoleModel, v []string) (tea.Cmd, error) { + if !strings.Contains(v[0], "/") { + return nil, fmt.Errorf("owner/name") + } + return m.call("Origin of "+md.Name, http.MethodPost, "/api/v1/models/"+url.PathEscape(md.Name)+"/origin", + map[string]string{"repo": v[0]}, func(raw json.RawMessage) string { + var o adminapi.OriginView + _ = json.Unmarshal(raw, &o) + return fmt.Sprintf("%s came from %s at %s; its updates are watched.", md.Name, o.Repo, + catalog.ShortRevision(o.Revision)) + }), nil + }}) + return nil + }} if md.Origin == nil { + if strings.Contains(md.Source, "://") { + return []menuItem{origin} + } return nil } server := "--server=" + m.client.base @@ -712,6 +738,7 @@ func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem { return fmt.Sprintf("%s at %s: %s", u.Repo, catalog.ShortRevision(u.Latest), updateCell(u)) }) }}) + items = append(items, origin) if prev := md.Origin.Previous; prev != "" { items = append(items, menuItem{"Go back to " + catalog.ShortRevision(prev) + "…", func(m *consoleModel) tea.Cmd { m.confirmD = &confirmDialog{title: "Go back on " + md.Name, diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index a32d851..fbf4db4 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -125,6 +125,7 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/updates/check", a.checkUpdates) mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel) mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel) + mux.HandleFunc("POST /api/v1/models/{name}/origin", a.setOrigin) mux.HandleFunc("GET /api/v1/requests/{id}", a.request) mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel) mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) { @@ -1520,6 +1521,29 @@ func (a *API) rollbackModel(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]string{"result": msg}) } +// OriginRequest names the repository a model's weights came from. +type OriginRequest struct { + Repo string `json:"repo"` +} + +func (a *API) setOrigin(w http.ResponseWriter, r *http.Request) { + if a.serve == nil { + writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one") + return + } + var req OriginRequest + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil || req.Repo == "" { + writeError(w, http.StatusBadRequest, "name the repository, owner/name") + return + } + o, err := a.serve.SetOriginOf(r.Context(), r.PathValue("name"), req.Repo) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, OriginView{Repo: o.Repo, Revision: o.Revision}) +} + // actorOf is who a change is recorded as made by: what the caller says, since // the admin API answers only this machine and trusts whoever reaches it. Kept // to a short printable name, so a claim cannot pass for more than one — no diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index 3ef4807..d68491d 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -146,6 +146,9 @@ type Server struct { seq int // updates is what the last checks of the registry found, by model. updates map[string]UpdateInfo + // origins are those worked out for models whose definition does not + // keep one, nil for a model that could not be traced. + origins map[string]*catalog.Origin } // New builds a server. @@ -162,7 +165,7 @@ func New(d Deps) *Server { if d.Log == nil { d.Log = slog.Default() } - return &Server{deps: d, jobs: map[string]*Job{}, updates: map[string]UpdateInfo{}} + return &Server{deps: d, jobs: map[string]*Job{}, updates: map[string]UpdateInfo{}, origins: map[string]*catalog.Origin{}} } var repoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) diff --git a/internal/control/modelserve/updates.go b/internal/control/modelserve/updates.go index cc446e8..62bebb6 100644 --- a/internal/control/modelserve/updates.go +++ b/internal/control/modelserve/updates.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/url" + "regexp" "slices" "strings" "sync/atomic" @@ -124,35 +125,156 @@ func originFiles(set hf.FileSet) []catalog.OriginFile { return out } -// originOf is a model's origin: recorded, or worked out from the mirror and -// then recorded. +// Finder is a registry that can be searched, and read at a past commit: what +// finding the origin of weights fetched before origins were kept needs. +type Finder interface { + Explore(ctx context.Context, q hf.Query) ([]hf.Model, error) + RepoAt(ctx context.Context, id, rev string) (hf.Repo, error) +} + +// originOf is a model's origin: recorded; or worked out from where the +// mirror fetched its files; or found on the registry by the files' names, +// the commit the mirror's directory is named after, and their sizes. What is +// worked out is recorded, so it is worked out once. func (s *Server) originOf(ctx context.Context, m catalog.Model, list mirror.ModelsResponse) *catalog.Origin { if m.Origin != nil && m.Origin.Repo != "" { return m.Origin } + s.mu.Lock() + o, known := s.origins[m.Name] + s.mu.Unlock() + if known { + return o + } p, ok := placeOf(m.Source, list) if !ok { return nil } - o, ok := inferOrigin(p) + o, ok = inferOrigin(p) if !ok { + o = s.findOrigin(ctx, p) + } + s.remember(ctx, m.Name, o) + return o +} + +// remember keeps an origin: in the catalog for a model added at runtime, and +// here for one from configuration, whose definition is not this one's to +// change. A nil origin is remembered too, so a model nobody can trace is not +// searched for at every check. +func (s *Server) remember(ctx context.Context, name string, o *catalog.Origin) { + s.mu.Lock() + s.origins[name] = o + s.mu.Unlock() + if o == nil || s.deps.Models == nil || s.deps.Models.Declared(name) { + return + } + if err := s.deps.Models.SetOrigin(ctx, name, o); err != nil { + s.deps.Log.Debug("could not record a model's origin", "model", name, "err", err) + } +} + +var hexRev = regexp.MustCompile(`^[0-9a-f]{7,40}$`) + +// findOrigin looks for the repository a mirror's files came from: one that +// holds a file of the same name, and has the commit the mirror's directory +// is named after with every file at the same size. Searched by the mirror's +// name for the model and the file's own; among the repositories that match, +// the most downloaded, which is the original rather than a copy of it. +func (s *Server) findOrigin(ctx context.Context, p place) *catalog.Origin { + f, ok := s.deps.Registry.(Finder) + if !ok || !hexRev.MatchString(p.version) { return nil } - if s.deps.Models != nil { - if err := s.deps.Models.SetOrigin(ctx, m.Name, o); err != nil { - s.deps.Log.Debug("could not record a model's origin", "model", m.Name, "err", err) + file := p.files[0].File + stem := strings.TrimSuffix(file, ".gguf") + if i := strings.LastIndexAny(stem, "-."); i > 0 { + stem = stem[:i] // the quantisation + } + tried := map[string]bool{} + for _, q := range []string{p.model, stem} { + found, err := f.Explore(ctx, hf.Query{Search: q, GGUF: true, Sort: hf.SortDownloads, Limit: 100, WithFiles: true}) + if err != nil { + s.deps.Log.Debug("searching the registry for a model's origin", "model", p.model, "err", err) + continue + } + for _, c := range found { + if tried[c.ID] || !slices.ContainsFunc(c.Siblings, func(sb struct { + RFilename string `json:"rfilename"` + }) bool { + return mirror.LastPathElement(sb.RFilename) == file + }) { + continue + } + tried[c.ID] = true + if o := s.verifyOrigin(ctx, f, c.ID, p); o != nil { + s.deps.Log.Info("found where a model's weights came from", "model", p.model, "repo", c.ID, + "revision", catalog.ShortRevision(o.Revision)) + return o + } } } + return nil +} + +// verifyOrigin reads a repository at the mirror's commit and says whether +// every file the mirror holds is there at the same size. +func (s *Server) verifyOrigin(ctx context.Context, f Finder, id string, p place) *catalog.Origin { + repo, err := f.RepoAt(ctx, id, p.version) + if err != nil || !strings.HasPrefix(repo.Revision, p.version) { + return nil + } + o := &catalog.Origin{Repo: repo.ID, Revision: repo.Revision} + for _, st := range p.files { + i := slices.IndexFunc(repo.Files, func(g hf.File) bool { + return mirror.LastPathElement(g.Path) == st.File && g.Bytes == st.BytesDone + }) + if i < 0 { + return nil + } + g := repo.Files[i] + o.Files = append(o.Files, catalog.OriginFile{Path: g.Path, SHA256: g.SHA256, Bytes: g.Bytes}) + } return o } +// SetOriginOf records, by hand, the repository a model's weights came from; +// the commit is the one the mirror's directory is named after, and the files +// must be there at the sizes the mirror holds. +func (s *Server) SetOriginOf(ctx context.Context, name, repoID string) (*catalog.Origin, error) { + m, err := s.current(name) + if err != nil { + return nil, err + } + list, err := s.deps.Mirror.List(ctx) + if err != nil { + return nil, fmt.Errorf("the mirror: %w", err) + } + p, ok := placeOf(m.Source, list) + if !ok { + return nil, fmt.Errorf("%s does not run from the mirror", name) + } + f, ok := s.deps.Registry.(Finder) + if !ok { + return nil, errors.New("this registry cannot be read at a past commit") + } + o := s.verifyOrigin(ctx, f, repoID, p) + if o == nil { + return nil, fmt.Errorf("%s has no commit %s holding %s at the size the mirror has", repoID, p.version, + p.files[0].File) + } + s.remember(ctx, name, o) + s.checkOne(ctx, m, o) + return o, nil +} + // compare says what the registry's repository has against an origin. func compare(m catalog.Model, o *catalog.Origin, repo hf.Repo) UpdateInfo { u := UpdateInfo{Model: m.Name, Repo: o.Repo, Current: o.Revision, Latest: repo.Revision, CheckedAt: time.Now()} if o.Previous != nil { u.Previous = o.Previous.Revision } - if repo.Revision == o.Revision { + if sameRevision(repo.Revision, o.Revision) { return u } for _, f := range o.Files { @@ -172,6 +294,14 @@ func compare(m catalog.Model, o *catalog.Origin, repo hf.Repo) UpdateInfo { return u } +// sameRevision compares commits either of which may be shortened. +func sameRevision(a, b string) bool { + if a == "" || b == "" { + return a == b + } + return strings.HasPrefix(a, b) || strings.HasPrefix(b, a) +} + // CheckUpdates asks the registry about every model with an origin. func (s *Server) CheckUpdates(ctx context.Context) { if s.deps.Current == nil { @@ -228,9 +358,16 @@ func (s *Server) CheckModel(ctx context.Context, name string) (UpdateInfo, error if err != nil { return UpdateInfo{}, fmt.Errorf("the mirror: %w", err) } + // Asked by hand: one that could not be traced is looked for again. + s.mu.Lock() + if o, ok := s.origins[name]; ok && o == nil { + delete(s.origins, name) + } + s.mu.Unlock() o := s.originOf(ctx, m, list) if o == nil { - return UpdateInfo{}, fmt.Errorf("%s did not come from the registry through the mirror, so there is nothing to compare", name) + return UpdateInfo{}, fmt.Errorf("where %s's weights came from is not known: give its repository with "+ + "`llmgw ctl model origin %s OWNER/REPO`", name, name) } return s.checkOne(ctx, m, o), nil } @@ -337,7 +474,7 @@ func (s *Server) runUpdate(j *Job) { s.mu.Unlock() u := compare(m, o, repo) switch { - case repo.Revision == o.Revision: + case sameRevision(repo.Revision, o.Revision): s.fail(j, "registry", fmt.Errorf("%s already runs %s's latest commit, %s", name, repo.ID, catalog.ShortRevision(repo.Revision))) return case len(u.Missing) > 0: diff --git a/internal/control/modelserve/updates_test.go b/internal/control/modelserve/updates_test.go index 104207f..4c586d8 100644 --- a/internal/control/modelserve/updates_test.go +++ b/internal/control/modelserve/updates_test.go @@ -2,6 +2,7 @@ package modelserve import ( "context" + "errors" "net/http" "net/http/httptest" "strings" @@ -262,3 +263,78 @@ func TestRegistryDownloadURLsAreRead(t *testing.T) { } } } + +// findingRegistry is a registry with history: searchable, and readable at a +// past commit. +type findingRegistry struct { + movingRegistry + found []hf.Model + at map[string]hf.Repo // id@rev +} + +func (r *findingRegistry) Explore(context.Context, hf.Query) ([]hf.Model, error) { return r.found, nil } +func (r *findingRegistry) RepoAt(_ context.Context, id, rev string) (hf.Repo, error) { + if repo, ok := r.at[id+"@"+rev]; ok { + return repo, nil + } + return hf.Repo{}, errors.New("404") +} + +func sibling(names ...string) []struct { + RFilename string `json:"rfilename"` +} { + var out []struct { + RFilename string `json:"rfilename"` + } + for _, n := range names { + out = append(out, struct { + RFilename string `json:"rfilename"` + }{n}) + } + return out +} + +func TestWeightsFetchedBeforeOriginsWereKeptAreTracedToTheirRepository(t *testing.T) { + s, mir, models, reg, _ := updSetup(t) + // The mirror kept neither source nor checksum. + mir.held[0].Source, mir.held[0].SHA256 = "", "" + fr := &findingRegistry{movingRegistry: movingRegistry{repo: reg.repo}, + found: []hf.Model{ + {ID: "copycat/r", Siblings: sibling("q4.gguf")}, // the right name, not the commit + {ID: "other/thing", Siblings: sibling("q8.gguf")}, // not the file + {ID: "o/r", Siblings: sibling("README.md", "q4.gguf")}, + }, + at: map[string]hf.Repo{ + "copycat/r@abc12345": {ID: "copycat/r", Revision: "ffff", Files: []hf.File{{Path: "q4.gguf", Bytes: 1 << 30}}}, + "o/r@abc12345": {ID: "o/r", Revision: oldRev, Files: []hf.File{ + {Path: "q4.gguf", SHA256: "aaaa", Bytes: 1 << 30}}}, + }} + s.deps.Registry = fr + s.CheckUpdates(context.Background()) + o := models.models["qwen"].Origin + if o == nil || o.Repo != "o/r" || o.Revision != oldRev || o.Files[0].SHA256 != "aaaa" { + t.Fatalf("traced to %+v", o) + } + if u := s.Updates()["qwen"]; !u.Available { + t.Errorf("%+v", u) + } +} + +func TestAnOriginGivenByHandIsChecked(t *testing.T) { + s, mir, models, reg, _ := updSetup(t) + mir.held[0].Source = "" + fr := &findingRegistry{movingRegistry: movingRegistry{repo: reg.repo}, at: map[string]hf.Repo{ + "o/r@abc12345": {ID: "o/r", Revision: oldRev, Files: []hf.File{{Path: "q4.gguf", SHA256: "aaaa", Bytes: 1 << 30}}}, + "bad/r@abc12345": {ID: "bad/r", Revision: oldRev, Files: []hf.File{{Path: "q4.gguf", Bytes: 5}}}, + }} + s.deps.Registry = fr + if _, err := s.SetOriginOf(context.Background(), "qwen", "bad/r"); err == nil { + t.Error("a repository whose file is another size was taken") + } + if _, err := s.SetOriginOf(context.Background(), "qwen", "o/r"); err != nil { + t.Fatal(err) + } + if o := models.models["qwen"].Origin; o == nil || o.Repo != "o/r" { + t.Fatalf("%+v", o) + } +} diff --git a/internal/hf/hf.go b/internal/hf/hf.go index db663f1..2c9edf4 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -68,6 +68,10 @@ type Model struct { // GGUF is what the registry read from the repository's weights, absent // for one that has none. GGUF *GGUFInfo `json:"gguf"` + // Siblings are the repository's files, when asked for with WithFiles. + Siblings []struct { + RFilename string `json:"rfilename"` + } `json:"siblings,omitempty"` } // GGUFInfo is the registry's summary of a repository's GGUF weights. @@ -160,6 +164,9 @@ type Query struct { // "lora", or "base_model:adapter:Qwen/Qwen2.5-7B-Instruct" for the // adapters made for one model. Filters []string + // WithFiles lists each repository's files too, to find one holding a + // file by name. + WithFiles bool } // exploreFields is everything a listing shows. Asking for any field with @@ -185,6 +192,9 @@ func (c *Client) Explore(ctx context.Context, q Query) ([]Model, error) { "direction": {"-1"}, "expand[]": exploreFields, } + if q.WithFiles { + v.Add("expand[]", "siblings") + } if s := strings.TrimSpace(q.Search); s != "" { v.Set("search", s) } @@ -330,7 +340,11 @@ func (r Repo) ShortRevision() string { } // Repo reads a repository's revision and file list. -func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { +func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { return c.RepoAt(ctx, id, "") } + +// RepoAt reads a repository as it was at a commit, which may be shortened; +// "" is its latest. +func (c *Client) RepoAt(ctx context.Context, id, rev string) (Repo, error) { id = strings.Trim(strings.TrimSpace(id), "/") if !strings.Contains(id, "/") { return Repo{}, fmt.Errorf("%q is not a repository; it looks like owner/name", id) @@ -351,7 +365,11 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { } // Expanded to what this reads: asking for gguf alone would leave out the // revision and the gate. - if err := c.get(ctx, "/api/models/"+id+"?expand[]=sha&expand[]=gated&expand[]=gguf&expand[]=cardData", &meta); err != nil { + at := "" + if rev != "" { + at = "/revision/" + url.PathEscape(rev) + } + if err := c.get(ctx, "/api/models/"+id+at+"?expand[]=sha&expand[]=gated&expand[]=gguf&expand[]=cardData", &meta); err != nil { return Repo{}, err } @@ -366,7 +384,7 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { } // By revision rather than by branch, so the listing and the URLs built from // it describe the same commit even if the branch moves in between. - rev := meta.SHA + rev = meta.SHA if rev == "" { rev = "main" } diff --git a/internal/mirror/add.go b/internal/mirror/add.go index 479e10a..a5d4fb5 100644 --- a/internal/mirror/add.go +++ b/internal/mirror/add.go @@ -101,6 +101,10 @@ func addWith(ctx context.Context, client *http.Client, root, source string, e En _ = os.Remove(part) return Added{}, err } + if err := writeSource(final, source); err != nil { + _ = os.Remove(part) + return Added{}, err + } if err := os.Rename(part, final); err != nil { return Added{}, err } @@ -156,6 +160,7 @@ func Remove(root string, e Entry) error { // Its checksum goes with it; left, it would keep the directory from // being empty, and so from going. _ = os.Remove(checksumPath(path)) + _ = os.Remove(sourcePath(path)) versionDir := filepath.Dir(path) if empty, _ := isEmpty(versionDir); empty { _ = os.Remove(versionDir) diff --git a/internal/mirror/manager.go b/internal/mirror/manager.go index a69752c..d85ef36 100644 --- a/internal/mirror/manager.go +++ b/internal/mirror/manager.go @@ -7,6 +7,7 @@ import ( "log/slog" "net/http" "os" + "path/filepath" "sync" "time" ) @@ -289,6 +290,10 @@ func (m *Manager) All() ([]Status, error) { // verifiable — the download finishing was what made it disappear. if k, ok := known[key]; ok { st.SHA256, st.Source = k.SHA256, k.Source + } else { + // Fetched before this process started: what was kept beside it. + path := filepath.Join(m.root, e.Model, e.Version, e.File) + st.SHA256, st.Source = readChecksum(path), readSource(path) } out = append(out, st) seen[key] = struct{}{} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 8b154e8..c2c3b4f 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -187,6 +187,31 @@ func writeChecksum(file, sum string) error { return os.Rename(tmp, checksumPath(file)) } +// sourcePath is where the URL a file was fetched from is kept, beside it like +// its checksum: what says, after the mirror restarts, which repository and +// commit a file is, so a newer one there can be noticed. +func sourcePath(file string) string { + return filepath.Join(filepath.Dir(file), "."+filepath.Base(file)+".source") +} + +func writeSource(file, source string) error { + tmp := sourcePath(file) + partSuffix + if err := os.WriteFile(tmp, []byte(source+"\n"), 0o644); err != nil { //nolint:gosec // a URL, for anyone to read + return err + } + return os.Rename(tmp, sourcePath(file)) +} + +// readSource is where a file was fetched from, or "" for one fetched before +// this was kept, or put in place by hand. +func readSource(file string) string { + b, err := os.ReadFile(sourcePath(file)) //nolint:gosec // beside a file in the mirror + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) +} + // readChecksum is a file's recorded sha256, or "" for a file put in place by // hand, which has none. func readChecksum(file string) string {