diff --git a/internal/cli/model.go b/internal/cli/model.go index 94e4f48..66c6d28 100644 --- a/internal/cli/model.go +++ b/internal/cli/model.go @@ -36,7 +36,7 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client), newTargetCmd(client, asJSON), newAdaptersCmd(client), newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client), - newModelRollbackCmd(client)) + newModelRollbackCmd(client), newModelOriginCmd(client)) return c } diff --git a/internal/cli/model_update.go b/internal/cli/model_update.go index 4111663..fe985dd 100644 --- a/internal/cli/model_update.go +++ b/internal/cli/model_update.go @@ -117,3 +117,25 @@ func newModelRollbackCmd(client func() *ctlClient) *cobra.Command { }, } } + +func newModelOriginCmd(client func() *ctlClient) *cobra.Command { + return &cobra.Command{ + Use: "origin MODEL OWNER/REPO", + Short: "Say which registry repository a model's weights came from", + Long: "For a model fetched before llmgw kept this, and that it could not find on\n" + + "its own. The repository is checked: it must have the commit the mirror's\n" + + "directory is named after, holding the model's files at the sizes the\n" + + "mirror has. From then on its updates are watched.", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + body, _ := json.Marshal(map[string]string{"repo": args[1]}) + var o struct{ Repo, Revision string } + if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/origin", body, &o); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "%s came from %s at %s; its updates are watched\n", args[0], o.Repo, + catalog.ShortRevision(o.Revision)) + return nil + }, + } +} diff --git a/internal/cli/tui_actions.go b/internal/cli/tui_actions.go index 05a0784..a4743c8 100644 --- a/internal/cli/tui_actions.go +++ b/internal/cli/tui_actions.go @@ -680,7 +680,33 @@ func (m *consoleModel) doEntry() tea.Cmd { // updateItems are what can be done about a model's repository publishing new // weights: move to them, ask now, or go back. func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem { + origin := menuItem{"Where its weights came from…", func(m *consoleModel) tea.Cmd { + repo := "" + if md.Origin != nil { + repo = md.Origin.Repo + } + m.openForm(&formDialog{title: "Origin of " + md.Name, + note: "The registry repository its files were fetched from. Checked: it must have the\n" + + "commit the mirror's directory is named after, with the same files at the same sizes.", + fields: []formField{textF("repository", repo, "owner/name, like bartowski/phi-4-GGUF")}, + submit: func(m *consoleModel, v []string) (tea.Cmd, error) { + if !strings.Contains(v[0], "/") { + return nil, fmt.Errorf("owner/name") + } + return m.call("Origin of "+md.Name, http.MethodPost, "/api/v1/models/"+url.PathEscape(md.Name)+"/origin", + map[string]string{"repo": v[0]}, func(raw json.RawMessage) string { + var o adminapi.OriginView + _ = json.Unmarshal(raw, &o) + return fmt.Sprintf("%s came from %s at %s; its updates are watched.", md.Name, o.Repo, + catalog.ShortRevision(o.Revision)) + }), nil + }}) + return nil + }} if md.Origin == nil { + if strings.Contains(md.Source, "://") { + return []menuItem{origin} + } return nil } server := "--server=" + m.client.base @@ -712,6 +738,7 @@ func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem { return fmt.Sprintf("%s at %s: %s", u.Repo, catalog.ShortRevision(u.Latest), updateCell(u)) }) }}) + items = append(items, origin) if prev := md.Origin.Previous; prev != "" { items = append(items, menuItem{"Go back to " + catalog.ShortRevision(prev) + "…", func(m *consoleModel) tea.Cmd { m.confirmD = &confirmDialog{title: "Go back on " + md.Name, diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index a32d851..fbf4db4 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -125,6 +125,7 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/updates/check", a.checkUpdates) mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel) mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel) + mux.HandleFunc("POST /api/v1/models/{name}/origin", a.setOrigin) mux.HandleFunc("GET /api/v1/requests/{id}", a.request) mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel) mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) { @@ -1520,6 +1521,29 @@ func (a *API) rollbackModel(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]string{"result": msg}) } +// OriginRequest names the repository a model's weights came from. +type OriginRequest struct { + Repo string `json:"repo"` +} + +func (a *API) setOrigin(w http.ResponseWriter, r *http.Request) { + if a.serve == nil { + writeError(w, http.StatusNotImplemented, "updates need a mirror and a store, and this control plane lacks one") + return + } + var req OriginRequest + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil || req.Repo == "" { + writeError(w, http.StatusBadRequest, "name the repository, owner/name") + return + } + o, err := a.serve.SetOriginOf(r.Context(), r.PathValue("name"), req.Repo) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusOK, OriginView{Repo: o.Repo, Revision: o.Revision}) +} + // actorOf is who a change is recorded as made by: what the caller says, since // the admin API answers only this machine and trusts whoever reaches it. Kept // to a short printable name, so a claim cannot pass for more than one — no diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index 3ef4807..d68491d 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -146,6 +146,9 @@ type Server struct { seq int // updates is what the last checks of the registry found, by model. updates map[string]UpdateInfo + // origins are those worked out for models whose definition does not + // keep one, nil for a model that could not be traced. + origins map[string]*catalog.Origin } // New builds a server. @@ -162,7 +165,7 @@ func New(d Deps) *Server { if d.Log == nil { d.Log = slog.Default() } - return &Server{deps: d, jobs: map[string]*Job{}, updates: map[string]UpdateInfo{}} + return &Server{deps: d, jobs: map[string]*Job{}, updates: map[string]UpdateInfo{}, origins: map[string]*catalog.Origin{}} } var repoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) diff --git a/internal/control/modelserve/updates.go b/internal/control/modelserve/updates.go index cc446e8..62bebb6 100644 --- a/internal/control/modelserve/updates.go +++ b/internal/control/modelserve/updates.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/url" + "regexp" "slices" "strings" "sync/atomic" @@ -124,35 +125,156 @@ func originFiles(set hf.FileSet) []catalog.OriginFile { return out } -// originOf is a model's origin: recorded, or worked out from the mirror and -// then recorded. +// Finder is a registry that can be searched, and read at a past commit: what +// finding the origin of weights fetched before origins were kept needs. +type Finder interface { + Explore(ctx context.Context, q hf.Query) ([]hf.Model, error) + RepoAt(ctx context.Context, id, rev string) (hf.Repo, error) +} + +// originOf is a model's origin: recorded; or worked out from where the +// mirror fetched its files; or found on the registry by the files' names, +// the commit the mirror's directory is named after, and their sizes. What is +// worked out is recorded, so it is worked out once. func (s *Server) originOf(ctx context.Context, m catalog.Model, list mirror.ModelsResponse) *catalog.Origin { if m.Origin != nil && m.Origin.Repo != "" { return m.Origin } + s.mu.Lock() + o, known := s.origins[m.Name] + s.mu.Unlock() + if known { + return o + } p, ok := placeOf(m.Source, list) if !ok { return nil } - o, ok := inferOrigin(p) + o, ok = inferOrigin(p) if !ok { + o = s.findOrigin(ctx, p) + } + s.remember(ctx, m.Name, o) + return o +} + +// remember keeps an origin: in the catalog for a model added at runtime, and +// here for one from configuration, whose definition is not this one's to +// change. A nil origin is remembered too, so a model nobody can trace is not +// searched for at every check. +func (s *Server) remember(ctx context.Context, name string, o *catalog.Origin) { + s.mu.Lock() + s.origins[name] = o + s.mu.Unlock() + if o == nil || s.deps.Models == nil || s.deps.Models.Declared(name) { + return + } + if err := s.deps.Models.SetOrigin(ctx, name, o); err != nil { + s.deps.Log.Debug("could not record a model's origin", "model", name, "err", err) + } +} + +var hexRev = regexp.MustCompile(`^[0-9a-f]{7,40}$`) + +// findOrigin looks for the repository a mirror's files came from: one that +// holds a file of the same name, and has the commit the mirror's directory +// is named after with every file at the same size. Searched by the mirror's +// name for the model and the file's own; among the repositories that match, +// the most downloaded, which is the original rather than a copy of it. +func (s *Server) findOrigin(ctx context.Context, p place) *catalog.Origin { + f, ok := s.deps.Registry.(Finder) + if !ok || !hexRev.MatchString(p.version) { return nil } - if s.deps.Models != nil { - if err := s.deps.Models.SetOrigin(ctx, m.Name, o); err != nil { - s.deps.Log.Debug("could not record a model's origin", "model", m.Name, "err", err) + file := p.files[0].File + stem := strings.TrimSuffix(file, ".gguf") + if i := strings.LastIndexAny(stem, "-."); i > 0 { + stem = stem[:i] // the quantisation + } + tried := map[string]bool{} + for _, q := range []string{p.model, stem} { + found, err := f.Explore(ctx, hf.Query{Search: q, GGUF: true, Sort: hf.SortDownloads, Limit: 100, WithFiles: true}) + if err != nil { + s.deps.Log.Debug("searching the registry for a model's origin", "model", p.model, "err", err) + continue + } + for _, c := range found { + if tried[c.ID] || !slices.ContainsFunc(c.Siblings, func(sb struct { + RFilename string `json:"rfilename"` + }) bool { + return mirror.LastPathElement(sb.RFilename) == file + }) { + continue + } + tried[c.ID] = true + if o := s.verifyOrigin(ctx, f, c.ID, p); o != nil { + s.deps.Log.Info("found where a model's weights came from", "model", p.model, "repo", c.ID, + "revision", catalog.ShortRevision(o.Revision)) + return o + } } } + return nil +} + +// verifyOrigin reads a repository at the mirror's commit and says whether +// every file the mirror holds is there at the same size. +func (s *Server) verifyOrigin(ctx context.Context, f Finder, id string, p place) *catalog.Origin { + repo, err := f.RepoAt(ctx, id, p.version) + if err != nil || !strings.HasPrefix(repo.Revision, p.version) { + return nil + } + o := &catalog.Origin{Repo: repo.ID, Revision: repo.Revision} + for _, st := range p.files { + i := slices.IndexFunc(repo.Files, func(g hf.File) bool { + return mirror.LastPathElement(g.Path) == st.File && g.Bytes == st.BytesDone + }) + if i < 0 { + return nil + } + g := repo.Files[i] + o.Files = append(o.Files, catalog.OriginFile{Path: g.Path, SHA256: g.SHA256, Bytes: g.Bytes}) + } return o } +// SetOriginOf records, by hand, the repository a model's weights came from; +// the commit is the one the mirror's directory is named after, and the files +// must be there at the sizes the mirror holds. +func (s *Server) SetOriginOf(ctx context.Context, name, repoID string) (*catalog.Origin, error) { + m, err := s.current(name) + if err != nil { + return nil, err + } + list, err := s.deps.Mirror.List(ctx) + if err != nil { + return nil, fmt.Errorf("the mirror: %w", err) + } + p, ok := placeOf(m.Source, list) + if !ok { + return nil, fmt.Errorf("%s does not run from the mirror", name) + } + f, ok := s.deps.Registry.(Finder) + if !ok { + return nil, errors.New("this registry cannot be read at a past commit") + } + o := s.verifyOrigin(ctx, f, repoID, p) + if o == nil { + return nil, fmt.Errorf("%s has no commit %s holding %s at the size the mirror has", repoID, p.version, + p.files[0].File) + } + s.remember(ctx, name, o) + s.checkOne(ctx, m, o) + return o, nil +} + // compare says what the registry's repository has against an origin. func compare(m catalog.Model, o *catalog.Origin, repo hf.Repo) UpdateInfo { u := UpdateInfo{Model: m.Name, Repo: o.Repo, Current: o.Revision, Latest: repo.Revision, CheckedAt: time.Now()} if o.Previous != nil { u.Previous = o.Previous.Revision } - if repo.Revision == o.Revision { + if sameRevision(repo.Revision, o.Revision) { return u } for _, f := range o.Files { @@ -172,6 +294,14 @@ func compare(m catalog.Model, o *catalog.Origin, repo hf.Repo) UpdateInfo { return u } +// sameRevision compares commits either of which may be shortened. +func sameRevision(a, b string) bool { + if a == "" || b == "" { + return a == b + } + return strings.HasPrefix(a, b) || strings.HasPrefix(b, a) +} + // CheckUpdates asks the registry about every model with an origin. func (s *Server) CheckUpdates(ctx context.Context) { if s.deps.Current == nil { @@ -228,9 +358,16 @@ func (s *Server) CheckModel(ctx context.Context, name string) (UpdateInfo, error if err != nil { return UpdateInfo{}, fmt.Errorf("the mirror: %w", err) } + // Asked by hand: one that could not be traced is looked for again. + s.mu.Lock() + if o, ok := s.origins[name]; ok && o == nil { + delete(s.origins, name) + } + s.mu.Unlock() o := s.originOf(ctx, m, list) if o == nil { - return UpdateInfo{}, fmt.Errorf("%s did not come from the registry through the mirror, so there is nothing to compare", name) + return UpdateInfo{}, fmt.Errorf("where %s's weights came from is not known: give its repository with "+ + "`llmgw ctl model origin %s OWNER/REPO`", name, name) } return s.checkOne(ctx, m, o), nil } @@ -337,7 +474,7 @@ func (s *Server) runUpdate(j *Job) { s.mu.Unlock() u := compare(m, o, repo) switch { - case repo.Revision == o.Revision: + case sameRevision(repo.Revision, o.Revision): s.fail(j, "registry", fmt.Errorf("%s already runs %s's latest commit, %s", name, repo.ID, catalog.ShortRevision(repo.Revision))) return case len(u.Missing) > 0: diff --git a/internal/control/modelserve/updates_test.go b/internal/control/modelserve/updates_test.go index 104207f..4c586d8 100644 --- a/internal/control/modelserve/updates_test.go +++ b/internal/control/modelserve/updates_test.go @@ -2,6 +2,7 @@ package modelserve import ( "context" + "errors" "net/http" "net/http/httptest" "strings" @@ -262,3 +263,78 @@ func TestRegistryDownloadURLsAreRead(t *testing.T) { } } } + +// findingRegistry is a registry with history: searchable, and readable at a +// past commit. +type findingRegistry struct { + movingRegistry + found []hf.Model + at map[string]hf.Repo // id@rev +} + +func (r *findingRegistry) Explore(context.Context, hf.Query) ([]hf.Model, error) { return r.found, nil } +func (r *findingRegistry) RepoAt(_ context.Context, id, rev string) (hf.Repo, error) { + if repo, ok := r.at[id+"@"+rev]; ok { + return repo, nil + } + return hf.Repo{}, errors.New("404") +} + +func sibling(names ...string) []struct { + RFilename string `json:"rfilename"` +} { + var out []struct { + RFilename string `json:"rfilename"` + } + for _, n := range names { + out = append(out, struct { + RFilename string `json:"rfilename"` + }{n}) + } + return out +} + +func TestWeightsFetchedBeforeOriginsWereKeptAreTracedToTheirRepository(t *testing.T) { + s, mir, models, reg, _ := updSetup(t) + // The mirror kept neither source nor checksum. + mir.held[0].Source, mir.held[0].SHA256 = "", "" + fr := &findingRegistry{movingRegistry: movingRegistry{repo: reg.repo}, + found: []hf.Model{ + {ID: "copycat/r", Siblings: sibling("q4.gguf")}, // the right name, not the commit + {ID: "other/thing", Siblings: sibling("q8.gguf")}, // not the file + {ID: "o/r", Siblings: sibling("README.md", "q4.gguf")}, + }, + at: map[string]hf.Repo{ + "copycat/r@abc12345": {ID: "copycat/r", Revision: "ffff", Files: []hf.File{{Path: "q4.gguf", Bytes: 1 << 30}}}, + "o/r@abc12345": {ID: "o/r", Revision: oldRev, Files: []hf.File{ + {Path: "q4.gguf", SHA256: "aaaa", Bytes: 1 << 30}}}, + }} + s.deps.Registry = fr + s.CheckUpdates(context.Background()) + o := models.models["qwen"].Origin + if o == nil || o.Repo != "o/r" || o.Revision != oldRev || o.Files[0].SHA256 != "aaaa" { + t.Fatalf("traced to %+v", o) + } + if u := s.Updates()["qwen"]; !u.Available { + t.Errorf("%+v", u) + } +} + +func TestAnOriginGivenByHandIsChecked(t *testing.T) { + s, mir, models, reg, _ := updSetup(t) + mir.held[0].Source = "" + fr := &findingRegistry{movingRegistry: movingRegistry{repo: reg.repo}, at: map[string]hf.Repo{ + "o/r@abc12345": {ID: "o/r", Revision: oldRev, Files: []hf.File{{Path: "q4.gguf", SHA256: "aaaa", Bytes: 1 << 30}}}, + "bad/r@abc12345": {ID: "bad/r", Revision: oldRev, Files: []hf.File{{Path: "q4.gguf", Bytes: 5}}}, + }} + s.deps.Registry = fr + if _, err := s.SetOriginOf(context.Background(), "qwen", "bad/r"); err == nil { + t.Error("a repository whose file is another size was taken") + } + if _, err := s.SetOriginOf(context.Background(), "qwen", "o/r"); err != nil { + t.Fatal(err) + } + if o := models.models["qwen"].Origin; o == nil || o.Repo != "o/r" { + t.Fatalf("%+v", o) + } +} diff --git a/internal/hf/hf.go b/internal/hf/hf.go index db663f1..2c9edf4 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -68,6 +68,10 @@ type Model struct { // GGUF is what the registry read from the repository's weights, absent // for one that has none. GGUF *GGUFInfo `json:"gguf"` + // Siblings are the repository's files, when asked for with WithFiles. + Siblings []struct { + RFilename string `json:"rfilename"` + } `json:"siblings,omitempty"` } // GGUFInfo is the registry's summary of a repository's GGUF weights. @@ -160,6 +164,9 @@ type Query struct { // "lora", or "base_model:adapter:Qwen/Qwen2.5-7B-Instruct" for the // adapters made for one model. Filters []string + // WithFiles lists each repository's files too, to find one holding a + // file by name. + WithFiles bool } // exploreFields is everything a listing shows. Asking for any field with @@ -185,6 +192,9 @@ func (c *Client) Explore(ctx context.Context, q Query) ([]Model, error) { "direction": {"-1"}, "expand[]": exploreFields, } + if q.WithFiles { + v.Add("expand[]", "siblings") + } if s := strings.TrimSpace(q.Search); s != "" { v.Set("search", s) } @@ -330,7 +340,11 @@ func (r Repo) ShortRevision() string { } // Repo reads a repository's revision and file list. -func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { +func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { return c.RepoAt(ctx, id, "") } + +// RepoAt reads a repository as it was at a commit, which may be shortened; +// "" is its latest. +func (c *Client) RepoAt(ctx context.Context, id, rev string) (Repo, error) { id = strings.Trim(strings.TrimSpace(id), "/") if !strings.Contains(id, "/") { return Repo{}, fmt.Errorf("%q is not a repository; it looks like owner/name", id) @@ -351,7 +365,11 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { } // Expanded to what this reads: asking for gguf alone would leave out the // revision and the gate. - if err := c.get(ctx, "/api/models/"+id+"?expand[]=sha&expand[]=gated&expand[]=gguf&expand[]=cardData", &meta); err != nil { + at := "" + if rev != "" { + at = "/revision/" + url.PathEscape(rev) + } + if err := c.get(ctx, "/api/models/"+id+at+"?expand[]=sha&expand[]=gated&expand[]=gguf&expand[]=cardData", &meta); err != nil { return Repo{}, err } @@ -366,7 +384,7 @@ func (c *Client) Repo(ctx context.Context, id string) (Repo, error) { } // By revision rather than by branch, so the listing and the URLs built from // it describe the same commit even if the branch moves in between. - rev := meta.SHA + rev = meta.SHA if rev == "" { rev = "main" } diff --git a/internal/mirror/add.go b/internal/mirror/add.go index 479e10a..a5d4fb5 100644 --- a/internal/mirror/add.go +++ b/internal/mirror/add.go @@ -101,6 +101,10 @@ func addWith(ctx context.Context, client *http.Client, root, source string, e En _ = os.Remove(part) return Added{}, err } + if err := writeSource(final, source); err != nil { + _ = os.Remove(part) + return Added{}, err + } if err := os.Rename(part, final); err != nil { return Added{}, err } @@ -156,6 +160,7 @@ func Remove(root string, e Entry) error { // Its checksum goes with it; left, it would keep the directory from // being empty, and so from going. _ = os.Remove(checksumPath(path)) + _ = os.Remove(sourcePath(path)) versionDir := filepath.Dir(path) if empty, _ := isEmpty(versionDir); empty { _ = os.Remove(versionDir) diff --git a/internal/mirror/manager.go b/internal/mirror/manager.go index a69752c..d85ef36 100644 --- a/internal/mirror/manager.go +++ b/internal/mirror/manager.go @@ -7,6 +7,7 @@ import ( "log/slog" "net/http" "os" + "path/filepath" "sync" "time" ) @@ -289,6 +290,10 @@ func (m *Manager) All() ([]Status, error) { // verifiable — the download finishing was what made it disappear. if k, ok := known[key]; ok { st.SHA256, st.Source = k.SHA256, k.Source + } else { + // Fetched before this process started: what was kept beside it. + path := filepath.Join(m.root, e.Model, e.Version, e.File) + st.SHA256, st.Source = readChecksum(path), readSource(path) } out = append(out, st) seen[key] = struct{}{} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 8b154e8..c2c3b4f 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -187,6 +187,31 @@ func writeChecksum(file, sum string) error { return os.Rename(tmp, checksumPath(file)) } +// sourcePath is where the URL a file was fetched from is kept, beside it like +// its checksum: what says, after the mirror restarts, which repository and +// commit a file is, so a newer one there can be noticed. +func sourcePath(file string) string { + return filepath.Join(filepath.Dir(file), "."+filepath.Base(file)+".source") +} + +func writeSource(file, source string) error { + tmp := sourcePath(file) + partSuffix + if err := os.WriteFile(tmp, []byte(source+"\n"), 0o644); err != nil { //nolint:gosec // a URL, for anyone to read + return err + } + return os.Rename(tmp, sourcePath(file)) +} + +// readSource is where a file was fetched from, or "" for one fetched before +// this was kept, or put in place by hand. +func readSource(file string) string { + b, err := os.ReadFile(sourcePath(file)) //nolint:gosec // beside a file in the mirror + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) +} + // readChecksum is a file's recorded sha256, or "" for a file put in place by // hand, which has none. func readChecksum(file string) string {