From dfbb7557ddc5f530e9c4c61d1440cc2c06c0cc15 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 27 Sep 2026 21:07:26 +0200 Subject: [PATCH] Serve a model made of several files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mirror would happily hold a four-shard model that nothing could serve. A catalog entry names one source, the agent fetched one URL, and so a version of several files had no way to be described — which is every safetensors model, and every GGUF too large to ship whole. A source ending in a slash names a version rather than a file. The agent fetches every file in it into one directory and hands the engine the directory: vLLM reads the shards and the config out of the folder, and llama.cpp is given the first part and finds the rest beside it. The trailing slash is the whole signal. A flag beside the source would be a second place for the two to disagree, and a URL already distinguishes a directory from a file everywhere else. The mirror answers a version with a manifest instead of a file listing. A plain file server gives HTML there, and something that has to know what a version consists of would be left parsing it. It is on the byte listener, not the management API: fetching weights is the one thing every node does, and making it depend on the administrative port would undo the split that keeps that port closeable. Built from the directory rather than a file written beside it. A manifest on disk is a second record of what is there, and the two disagree the first time somebody copies a file in by hand — which the mirror explicitly allows. The whole set or nothing. A directory with three of four shards loads as a broken model rather than failing to load, so a completion marker is written last and an interrupted fetch starts the missing files again rather than being found and served. Files already there at the right size are not fetched twice. Progress is reported under the version's address, not each file's: the shards waiting on it asked for the version, and progress under a file's own URL reaches nobody. A filename from the manifest is refused if it contains a separator. The mirror is a place an operator copies files into by hand, so what it reports is not automatically safe to join onto a path. An HTML listing where a manifest was expected says so. "invalid character '<'" sends somebody looking at the wrong thing. Co-Authored-By: Claude Opus 5 (1M context) --- internal/agent/engines/llamacpp.go | 44 +++++- internal/agent/engines/llamacpp_test.go | 58 ++++++++ internal/agent/weights/fetch.go | 18 ++- internal/agent/weights/set.go | 129 +++++++++++++++++ internal/agent/weights/set_test.go | 167 +++++++++++++++++++++++ internal/cli/mirror.go | 47 ++++++- internal/mirror/mirror.go | 44 ++++++ internal/web/admin_serve.go | 30 ++-- internal/web/admin_serve_test.go | 36 +++-- internal/web/templates/admin_mirror.html | 4 +- 10 files changed, 554 insertions(+), 23 deletions(-) create mode 100644 internal/agent/weights/set.go create mode 100644 internal/agent/weights/set_test.go diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index 1f411e85..775b77e2 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -2,6 +2,9 @@ package engines import ( "fmt" + "os" + "path/filepath" + "sort" "strconv" "strings" ) @@ -71,8 +74,12 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { } p := s.Profile + model, err := ggufIn(p.ModelPath) + if err != nil { + return nil, err + } args := []string{ - "--model", p.ModelPath, + "--model", model, "--host", "0.0.0.0", "--port", strconv.Itoa(s.HTTPPort), // Without this there is no /metrics, and an endpoint the router cannot @@ -127,3 +134,38 @@ func hasFlag(args []string, names ...string) bool { } return false } + +// ggufIn resolves a model path that may be a directory. +// +// A multi-file version arrives as the directory it was downloaded into, because +// that is the only shape a set can take. llama.cpp wants a file: given the +// first part of a split GGUF it finds the rest beside it, so the first part is +// what it gets. +// +// A path that is already a file is returned untouched, which is every +// single-file model and every hand-staged one. +func ggufIn(path string) (string, error) { + st, err := os.Stat(path) + if err != nil || !st.IsDir() { + // Not a directory, or not there at all — the second is the engine's to + // report, with the message it would have given anyway. + return path, nil //nolint:nilerr // a missing path is the engine's error to give + } + entries, err := os.ReadDir(path) + if err != nil { + return "", err + } + var names []string + for _, e := range entries { + if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), ".gguf") { + names = append(names, e.Name()) + } + } + if len(names) == 0 { + return "", fmt.Errorf("%s holds no .gguf file", path) + } + // Sorted, so a split model's first part comes first: the parts are named + // "-00001-of-00004" and lexical order is shard order. + sort.Strings(names) + return filepath.Join(path, names[0]), nil +} diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go index 30419ee6..7e93354a 100644 --- a/internal/agent/engines/llamacpp_test.go +++ b/internal/agent/engines/llamacpp_test.go @@ -1,6 +1,8 @@ package engines import ( + "os" + "path/filepath" "slices" "strings" "testing" @@ -187,3 +189,59 @@ func TestLlamaCppLetsAProfileTurnTheTemplateOff(t *testing.T) { t.Errorf("the profile said --no-jinja and got both: %v", cmds[0].Args) } } + +func TestASetDirectoryResolvesToItsFirstPart(t *testing.T) { + dir := t.TempDir() + // Deliberately out of order on disk, because readdir order is not shard + // order and the first part is the one llama.cpp must be given. + for _, n := range []string{ + "qwen-q4_k_m-00003-of-00003.gguf", + "qwen-q4_k_m-00001-of-00003.gguf", + "qwen-q4_k_m-00002-of-00003.gguf", + } { + if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + got, err := ggufIn(dir) + if err != nil { + t.Fatal(err) + } + // Given the first part llama.cpp finds the rest beside it; given the third + // it loads a third of a model. + if filepath.Base(got) != "qwen-q4_k_m-00001-of-00003.gguf" { + t.Errorf("expected the first shard, got %s", filepath.Base(got)) + } +} + +func TestAPlainFilePathIsUntouched(t *testing.T) { + // Every single-file model and every hand-staged one takes this path. + p := filepath.Join(t.TempDir(), "model.gguf") + if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if got, _ := ggufIn(p); got != p { + t.Errorf("expected the path unchanged, got %s", got) + } +} + +func TestADirectoryWithNoWeightsSaysSo(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := ggufIn(dir); err == nil { + t.Error("a directory with no .gguf in it is not a model llama.cpp can load") + } +} + +func TestAMissingPathIsLeftToTheEngine(t *testing.T) { + // The engine's own "no such file" names the path and the flag it came + // from, which is more use than anything this could say. + p := "/nonexistent/model.gguf" + got, err := ggufIn(p) + if err != nil || got != p { + t.Errorf("expected it passed through, got %q %v", got, err) + } +} diff --git a/internal/agent/weights/fetch.go b/internal/agent/weights/fetch.go index 09ea796c..8813e348 100644 --- a/internal/agent/weights/fetch.go +++ b/internal/agent/weights/fetch.go @@ -84,6 +84,11 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) { if source == "" { return "", fmt.Errorf("no source given") } + // A version rather than a file. Everything below assumes one URL is one + // file, and a set has to be handled before any of it. + if IsSet(source) { + return f.ensureSet(ctx, source) + } if !isRemote(source) { if !isPath(source) { // A bare model id — "Qwen/Qwen2.5-7B-Instruct" — is not a file and @@ -134,6 +139,15 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) { // fetch downloads a source to a target, resuming and verifying. func (f *Fetcher) fetch(ctx context.Context, source, target string) (string, error) { + return f.fetchAs(ctx, source, source, target) +} + +// fetchAs downloads one URL but reports progress under another name. +// +// The two differ for a set: the shards waiting on it asked for the version, not +// for the fourth file in it, and progress reported under the file's own URL +// reaches nobody. What they see is one number climbing across the whole set. +func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string) (string, error) { url, err := resolveURL(source) if err != nil { return "", err @@ -192,8 +206,8 @@ func (f *Fetcher) fetch(ctx context.Context, source, target string) (string, err } total := existing + resp.ContentLength - f.report(Progress{Source: source, BytesDone: existing, BytesTotal: total}) - written, err := f.copyWithProgress(ctx, out, resp.Body, source, existing, total) + f.report(Progress{Source: reportAs, BytesDone: existing, BytesTotal: total}) + written, err := f.copyWithProgress(ctx, out, resp.Body, reportAs, existing, total) closeErr := out.Close() if err != nil { // The part file is kept on purpose, so the next attempt resumes. diff --git a/internal/agent/weights/set.go b/internal/agent/weights/set.go new file mode 100644 index 00000000..aaba4e1e --- /dev/null +++ b/internal/agent/weights/set.go @@ -0,0 +1,129 @@ +package weights + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" +) + +// IsSet reports whether a source names a set of files rather than one. +// +// By the trailing slash, which is the whole signal: "…/v1/model.gguf" is a file +// and "…/v1/" is the version it belongs to. A flag beside the source would be a +// second place for the two to disagree, and the URL already distinguishes a +// directory from a file everywhere else on the web. +func IsSet(source string) bool { + return isRemote(source) && strings.HasSuffix(source, "/") +} + +// manifest is what the mirror answers for a version. +type manifest struct { + Model string `json:"model"` + Version string `json:"version"` + Files []struct { + Name string `json:"name"` + Bytes int64 `json:"bytes"` + } `json:"files"` + Bytes int64 `json:"bytes"` +} + +// ensureSet downloads every file of a version into one directory. +// +// A directory, because that is what a multi-file model is: vLLM is handed the +// folder and reads the shards and the config out of it, and llama.cpp is handed +// the first part and finds the rest beside it. Downloading them into one place +// is what makes either possible, and a catalog entry naming one file could +// express neither. +// +// The whole set or nothing. A directory with three of four shards in it loads +// as a broken model rather than failing to load, so a partial set is left +// unfinished under its temporary name and the next attempt starts again. +func (f *Fetcher) ensureSet(ctx context.Context, source string) (string, error) { + dir, err := f.cachePath(strings.TrimSuffix(source, "/")) + if err != nil { + return "", err + } + // A complete set is marked by a file written last. Checking for the + // directory alone would treat an interrupted download as finished, which is + // exactly the failure the marker exists to prevent. + done := filepath.Join(dir, ".complete") + if _, err := os.Stat(done); err == nil { + return dir, nil + } + + m, err := f.manifestFor(ctx, source) + if err != nil { + return "", err + } + if len(m.Files) == 0 { + return "", fmt.Errorf("%s holds no files", source) + } + + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + for _, file := range m.Files { + // Rejected rather than joined: a name with a separator in it would + // write outside the directory the mirror said it was in, and the + // mirror is a place an operator copies files into by hand. + if strings.ContainsAny(file.Name, `/\`) || file.Name == ".." { + return "", fmt.Errorf("%s: the mirror offered a file named %q", source, file.Name) + } + target := filepath.Join(dir, file.Name) + if st, err := os.Stat(target); err == nil && st.Size() == file.Bytes { + continue + } + // Reported under the set's own address, because that is what the + // shards are waiting on. Reusing fetch rather than writing a second + // downloader keeps resume, the .part rename and the progress callback + // in one place. + if _, err := f.fetchAs(ctx, source+file.Name, source, target); err != nil { + return "", err + } + } + + if err := os.WriteFile(done, []byte(source+"\n"), 0o644); err != nil { + return "", err + } + return dir, nil +} + +// manifestFor asks the mirror what a version consists of. +func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil) + if err != nil { + return manifest{}, err + } + client := f.Client + if client == nil { + client = &http.Client{} + } + resp, err := client.Do(req) + if err != nil { + return manifest{}, fmt.Errorf("asking %s what it holds: %w", source, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return manifest{}, fmt.Errorf("%s answered %d", source, resp.StatusCode) + } + + var m manifest + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return manifest{}, err + } + if err := json.Unmarshal(body, &m); err != nil { + // The likely cause is a plain file server with a directory index, which + // answers HTML. Said plainly, because "invalid character '<'" sends + // somebody looking at the wrong thing. + return manifest{}, fmt.Errorf( + "%s did not answer with a manifest; a source ending in / has to be a "+ + "mirror version, not a directory listing", source) + } + return m, nil +} diff --git a/internal/agent/weights/set_test.go b/internal/agent/weights/set_test.go new file mode 100644 index 00000000..4a1e167b --- /dev/null +++ b/internal/agent/weights/set_test.go @@ -0,0 +1,167 @@ +package weights + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +// mirrorOf serves a manifest at "/m/v/" and the files beneath it. +func mirrorOf(t *testing.T, files map[string]string) (*Fetcher, *httptest.Server, *int) { + t.Helper() + hits := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/") { + var b strings.Builder + b.WriteString(`{"model":"m","version":"v","files":[`) + first := true + total := 0 + for name, body := range files { + if !first { + b.WriteString(",") + } + first = false + fmt.Fprintf(&b, `{"name":%q,"bytes":%d}`, name, len(body)) + total += len(body) + } + fmt.Fprintf(&b, `],"bytes":%d}`, total) + _, _ = w.Write([]byte(b.String())) + return + } + body, ok := files[strings.TrimPrefix(r.URL.Path, "/m/v/")] + if !ok { + http.NotFound(w, r) + return + } + hits++ + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return &Fetcher{CacheDir: t.TempDir()}, srv, &hits +} + +func TestASetIsFetchedIntoOneDirectory(t *testing.T) { + f, srv, _ := mirrorOf(t, map[string]string{ + "model-00001-of-00002.safetensors": "aaaa", + "model-00002-of-00002.safetensors": "bb", + "config.json": "{}", + }) + + got, err := f.Ensure(t.Context(), srv.URL+"/m/v/") + if err != nil { + t.Fatal(err) + } + st, err := os.Stat(got) + if err != nil || !st.IsDir() { + t.Fatalf("expected a directory, got %q: %v", got, err) + } + // A directory, because that is what a multi-file model is: vLLM reads the + // shards and the config out of the folder, and a catalog entry naming one + // file could express neither. + for _, name := range []string{"model-00001-of-00002.safetensors", "config.json"} { + if _, err := os.Stat(filepath.Join(got, name)); err != nil { + t.Errorf("%s did not arrive: %v", name, err) + } + } +} + +func TestAFinishedSetIsNotFetchedAgain(t *testing.T) { + f, srv, hits := mirrorOf(t, map[string]string{"a.gguf": "aaaa", "b.gguf": "bb"}) + + if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil { + t.Fatal(err) + } + after := *hits + if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil { + t.Fatal(err) + } + if *hits != after { + t.Errorf("the second call downloaded again: %d then %d", after, *hits) + } +} + +func TestAHalfFetchedSetIsNotTreatedAsFinished(t *testing.T) { + f, srv, _ := mirrorOf(t, map[string]string{"a.gguf": "aaaa", "b.gguf": "bb"}) + dir, err := f.cachePath(srv.URL + "/m/v") + if err != nil { + t.Fatal(err) + } + // One file present, no completion marker — an interrupted download. + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "a.gguf"), []byte("aaaa"), 0o644); err != nil { + t.Fatal(err) + } + + got, err := f.Ensure(t.Context(), srv.URL+"/m/v/") + if err != nil { + t.Fatal(err) + } + // A directory with three of four shards loads as a broken model rather + // than failing to load, so the marker is what says the set is whole. + if _, err := os.Stat(filepath.Join(got, "b.gguf")); err != nil { + t.Errorf("the missing file was not fetched: %v", err) + } +} + +func TestAFileAlreadyThereAtTheRightSizeIsNotRefetched(t *testing.T) { + f, srv, hits := mirrorOf(t, map[string]string{"a.gguf": "aaaa", "b.gguf": "bb"}) + dir, _ := f.cachePath(srv.URL + "/m/v") + _ = os.MkdirAll(dir, 0o755) + _ = os.WriteFile(filepath.Join(dir, "a.gguf"), []byte("aaaa"), 0o644) + + if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil { + t.Fatal(err) + } + // Resuming a set means not paying for the shards that already landed. + if *hits != 1 { + t.Errorf("expected only the missing file to be fetched, got %d downloads", *hits) + } +} + +func TestAFilenameThatEscapesTheDirectoryIsRefused(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"files":[{"name":"../../etc/passwd","bytes":1}]}`)) + })) + defer srv.Close() + f := &Fetcher{CacheDir: t.TempDir()} + + _, err := f.Ensure(t.Context(), srv.URL+"/m/v/") + // The mirror is a place an operator copies files into by hand, so what it + // reports is not automatically safe to join onto a path. + if err == nil || !strings.Contains(err.Error(), "offered a file named") { + t.Errorf("expected the name to be refused, got %v", err) + } +} + +func TestADirectoryListingIsNotAManifest(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("
a.gguf
")) + })) + defer srv.Close() + f := &Fetcher{CacheDir: t.TempDir()} + + _, err := f.Ensure(t.Context(), srv.URL+"/m/v/") + // "invalid character '<'" sends somebody looking at the wrong thing. + if err == nil || !strings.Contains(err.Error(), "has to be a mirror version") { + t.Errorf("the reason should name the cause, got %v", err) + } +} + +func TestOnlyATrailingSlashMeansASet(t *testing.T) { + for src, want := range map[string]bool{ + "http://mirror/m/v/": true, + "http://mirror/m/v/model.gguf": false, + "/var/lib/models/": false, // a local directory is not fetched + "Qwen/Qwen2.5-7B-Instruct": false, // a repo id, resolved by the engine + } { + if got := IsSet(src); got != want { + t.Errorf("IsSet(%q) = %v, want %v", src, got, want) + } + } +} diff --git a/internal/cli/mirror.go b/internal/cli/mirror.go index 84be076c..934f633d 100644 --- a/internal/cli/mirror.go +++ b/internal/cli/mirror.go @@ -1,6 +1,7 @@ package cli import ( + "encoding/json" "fmt" "log/slog" "net/http" @@ -264,7 +265,7 @@ func newMirrorServeCmd(dir *string) *cobra.Command { // by me. files := &http.Server{ Addr: listen, - Handler: readOnly(http.FileServer(http.Dir(*dir))), + Handler: readOnly(withManifest(*dir, http.FileServer(http.Dir(*dir)))), ReadHeaderTimeout: 10 * time.Second, } admin := &http.Server{ @@ -353,3 +354,47 @@ func catalogSources(cmd *cobra.Command, server string) ([]string, error) { } return out, nil } + +// withManifest answers a request for a version with the list of files in it. +// +// A directory otherwise gets FileServer's HTML index, and something that has to +// know what a version consists of would be left parsing it. A model is +// routinely several files — sharded safetensors, or a GGUF split across parts — +// and a catalog entry names one source, so without a way to ask "what is in +// this version" a sharded model can be mirrored and never served. +// +// On the byte listener rather than the management API on purpose. Fetching +// weights is the one thing every node does, and making it depend on the +// administrative port would undo the split that keeps that port closeable. +func withManifest(root string, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + model, version, ok := versionPath(r.URL.Path) + if !ok { + next.ServeHTTP(w, r) + return + } + m, err := mirror.ManifestFor(root, model, version) + if err != nil { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(m) + }) +} + +// versionPath matches "/model/version/" and nothing else. +// +// The trailing slash is the whole signal: "/m/v/file.gguf" is a file and +// "/m/v/" is the set it belongs to. Requiring it means a source URL says which +// of the two it means without a flag beside it. +func versionPath(p string) (model, version string, ok bool) { + if !strings.HasSuffix(p, "/") { + return "", "", false + } + parts := strings.Split(strings.Trim(p, "/"), "/") + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", false + } + return parts[0], parts[1], true +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index cf8f1fb8..75b2d198 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -172,3 +172,47 @@ func sha256File(path string) (string, error) { } return hex.EncodeToString(h.Sum(nil)), nil } + +// Manifest is what one version of a model consists of. +// +// A version is a directory and a model is routinely several files — sharded +// safetensors, or a GGUF split across parts. A catalog entry names one source, +// so without something that says "these files together are the model" a sharded +// model can be mirrored and never served. +type Manifest struct { + Model string `json:"model"` + Version string `json:"version"` + Files []ManifestFile `json:"files"` + Bytes int64 `json:"bytes"` +} + +// ManifestFile is one file of a version. +type ManifestFile struct { + Name string `json:"name"` + Bytes int64 `json:"bytes"` +} + +// ManifestFor lists the files of one model version. +// +// Built from the directory rather than from a file written alongside it. A +// manifest on disk is a second record of what is there, and the two disagree +// the first time somebody copies a file in by hand — which is a thing the +// mirror is explicitly meant to allow. +func ManifestFor(root, model, version string) (Manifest, error) { + entries, err := List(root) + if err != nil { + return Manifest{}, err + } + m := Manifest{Model: model, Version: version} + for _, e := range entries { + if e.Model != model || e.Version != version { + continue + } + m.Files = append(m.Files, ManifestFile{Name: e.File, Bytes: e.Bytes}) + m.Bytes += e.Bytes + } + if len(m.Files) == 0 { + return Manifest{}, fmt.Errorf("%s/%s: %w", model, version, os.ErrNotExist) + } + return m, nil +} diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index ea7836d3..0a910769 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -32,6 +32,8 @@ type serveForm struct { // be written later. Named for the hardware rather than the model, because // a profile is about where a model runs. Profile string + // Files is how many files the version holds, above one for a set. + Files int // MinMemoryGB is the host RAM to ask for, estimated from the weights. // // Filled in rather than left blank. Blank writes no constraint at all, and @@ -63,16 +65,26 @@ func planServe(st mirror.Status, files []mirror.Status, base string, Profile: "cpu", } - // A catalogue entry names one source. A version made of several files has - // no single one, so an entry for it would be an entry for one shard — which - // loads as a broken model rather than failing to load. + // A version of several files is served as the version, not as one of them. + // The source is the directory — a trailing slash — and the agent fetches + // every file in it and hands the engine the folder. if len(files) > 1 { - f.Refusal = fmt.Sprintf( - "this version is %d files, and a catalogue entry names one source. "+ - "Serve a single-file build (a GGUF), or give the engine the "+ - "repository id so it fetches the shards itself", - len(files)) - return f + f.Source = strings.TrimSuffix( + mirror.Entry{Model: st.Model, Version: st.Version}.URL(base), "/") + "/" + f.Files = len(files) + // From whichever file says, because a version is a mixture only when + // somebody has put one there by hand. The weights decide the engine. + for _, one := range files { + if e := engineFor(one.File); e != "" { + f.Engine = e + break + } + } + var total int64 + for _, one := range files { + total += one.BytesDone + } + st.BytesTotal = total } switch f.Engine { diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go index e0fb9475..a048b49f 100644 --- a/internal/web/admin_serve_test.go +++ b/internal/web/admin_serve_test.go @@ -76,22 +76,40 @@ func TestAGGUFCanBeServedStraightFromTheList(t *testing.T) { } } -func TestAShardedVersionIsRefusedWithTheReason(t *testing.T) { +func TestAShardedVersionIsServedAsTheWholeVersion(t *testing.T) { models := []mirror.Status{ held("llama", "v1", "model-00001-of-00002.safetensors"), held("llama", "v1", "model-00002-of-00002.safetensors"), } - body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8), + body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8, "vllm"), "/admin/mirror?serve=llama:v1").Body.String() - // A catalogue entry names one source, so an entry for a sharded version is - // an entry for one shard — which loads as a broken model rather than - // failing to load. - if !strings.Contains(body, "a catalogue entry names one source") { - t.Errorf("the refusal should say why; got %s", body) + // A catalogue entry names one source, and for a version of several files + // that source is the version. Refusing it meant a sharded model could be + // mirrored and never served — which is most of what is not a GGUF. + if !strings.Contains(body, `value="http://mirror:8081/llama/v1/"`) { + t.Errorf("the source should be the version directory; got %s", body) } - if strings.Contains(body, `action="/admin/mirror/serve"`) { - t.Error("no form should be offered for something that cannot be served") + if !strings.Contains(body, "The whole version, 2 files") { + t.Error("the form should say it is fetching a set") + } + if !strings.Contains(body, `action="/admin/mirror/serve"`) { + t.Error("it should be servable") + } +} + +func TestTheEngineForASetComesFromItsWeights(t *testing.T) { + // A version is a mixture only when somebody has put a file there by hand. + // config.json decides nothing; the safetensors do. + models := []mirror.Status{ + held("llama", "v1", "config.json"), + held("llama", "v1", "model-00001-of-00002.safetensors"), + } + body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8, "vllm"), + "/admin/mirror?serve=llama:v1").Body.String() + + if !strings.Contains(body, `name="engine" value="vllm"`) { + t.Errorf("the weights should pick the engine; got %s", body) } } diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index 0d4fbbce..f6cac445 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -93,7 +93,9 @@ reachable from this machine.

-

Serving {{.Source}} with {{.Engine}}.

+

Serving {{.Source}} with {{.Engine}}. + {{if .Files}}The whole version, {{.Files}} files: the node fetches them into one + directory and the engine is given the directory.{{end}}