diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index 1f411e85..775b77e2 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -2,6 +2,9 @@ package engines import ( "fmt" + "os" + "path/filepath" + "sort" "strconv" "strings" ) @@ -71,8 +74,12 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { } p := s.Profile + model, err := ggufIn(p.ModelPath) + if err != nil { + return nil, err + } args := []string{ - "--model", p.ModelPath, + "--model", model, "--host", "0.0.0.0", "--port", strconv.Itoa(s.HTTPPort), // Without this there is no /metrics, and an endpoint the router cannot @@ -127,3 +134,38 @@ func hasFlag(args []string, names ...string) bool { } return false } + +// ggufIn resolves a model path that may be a directory. +// +// A multi-file version arrives as the directory it was downloaded into, because +// that is the only shape a set can take. llama.cpp wants a file: given the +// first part of a split GGUF it finds the rest beside it, so the first part is +// what it gets. +// +// A path that is already a file is returned untouched, which is every +// single-file model and every hand-staged one. +func ggufIn(path string) (string, error) { + st, err := os.Stat(path) + if err != nil || !st.IsDir() { + // Not a directory, or not there at all — the second is the engine's to + // report, with the message it would have given anyway. + return path, nil //nolint:nilerr // a missing path is the engine's error to give + } + entries, err := os.ReadDir(path) + if err != nil { + return "", err + } + var names []string + for _, e := range entries { + if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), ".gguf") { + names = append(names, e.Name()) + } + } + if len(names) == 0 { + return "", fmt.Errorf("%s holds no .gguf file", path) + } + // Sorted, so a split model's first part comes first: the parts are named + // "-00001-of-00004" and lexical order is shard order. + sort.Strings(names) + return filepath.Join(path, names[0]), nil +} diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go index 30419ee6..7e93354a 100644 --- a/internal/agent/engines/llamacpp_test.go +++ b/internal/agent/engines/llamacpp_test.go @@ -1,6 +1,8 @@ package engines import ( + "os" + "path/filepath" "slices" "strings" "testing" @@ -187,3 +189,59 @@ func TestLlamaCppLetsAProfileTurnTheTemplateOff(t *testing.T) { t.Errorf("the profile said --no-jinja and got both: %v", cmds[0].Args) } } + +func TestASetDirectoryResolvesToItsFirstPart(t *testing.T) { + dir := t.TempDir() + // Deliberately out of order on disk, because readdir order is not shard + // order and the first part is the one llama.cpp must be given. + for _, n := range []string{ + "qwen-q4_k_m-00003-of-00003.gguf", + "qwen-q4_k_m-00001-of-00003.gguf", + "qwen-q4_k_m-00002-of-00003.gguf", + } { + if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + got, err := ggufIn(dir) + if err != nil { + t.Fatal(err) + } + // Given the first part llama.cpp finds the rest beside it; given the third + // it loads a third of a model. + if filepath.Base(got) != "qwen-q4_k_m-00001-of-00003.gguf" { + t.Errorf("expected the first shard, got %s", filepath.Base(got)) + } +} + +func TestAPlainFilePathIsUntouched(t *testing.T) { + // Every single-file model and every hand-staged one takes this path. + p := filepath.Join(t.TempDir(), "model.gguf") + if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if got, _ := ggufIn(p); got != p { + t.Errorf("expected the path unchanged, got %s", got) + } +} + +func TestADirectoryWithNoWeightsSaysSo(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := ggufIn(dir); err == nil { + t.Error("a directory with no .gguf in it is not a model llama.cpp can load") + } +} + +func TestAMissingPathIsLeftToTheEngine(t *testing.T) { + // The engine's own "no such file" names the path and the flag it came + // from, which is more use than anything this could say. + p := "/nonexistent/model.gguf" + got, err := ggufIn(p) + if err != nil || got != p { + t.Errorf("expected it passed through, got %q %v", got, err) + } +} diff --git a/internal/agent/weights/fetch.go b/internal/agent/weights/fetch.go index 09ea796c..8813e348 100644 --- a/internal/agent/weights/fetch.go +++ b/internal/agent/weights/fetch.go @@ -84,6 +84,11 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) { if source == "" { return "", fmt.Errorf("no source given") } + // A version rather than a file. Everything below assumes one URL is one + // file, and a set has to be handled before any of it. + if IsSet(source) { + return f.ensureSet(ctx, source) + } if !isRemote(source) { if !isPath(source) { // A bare model id — "Qwen/Qwen2.5-7B-Instruct" — is not a file and @@ -134,6 +139,15 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) { // fetch downloads a source to a target, resuming and verifying. func (f *Fetcher) fetch(ctx context.Context, source, target string) (string, error) { + return f.fetchAs(ctx, source, source, target) +} + +// fetchAs downloads one URL but reports progress under another name. +// +// The two differ for a set: the shards waiting on it asked for the version, not +// for the fourth file in it, and progress reported under the file's own URL +// reaches nobody. What they see is one number climbing across the whole set. +func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string) (string, error) { url, err := resolveURL(source) if err != nil { return "", err @@ -192,8 +206,8 @@ func (f *Fetcher) fetch(ctx context.Context, source, target string) (string, err } total := existing + resp.ContentLength - f.report(Progress{Source: source, BytesDone: existing, BytesTotal: total}) - written, err := f.copyWithProgress(ctx, out, resp.Body, source, existing, total) + f.report(Progress{Source: reportAs, BytesDone: existing, BytesTotal: total}) + written, err := f.copyWithProgress(ctx, out, resp.Body, reportAs, existing, total) closeErr := out.Close() if err != nil { // The part file is kept on purpose, so the next attempt resumes. diff --git a/internal/agent/weights/set.go b/internal/agent/weights/set.go new file mode 100644 index 00000000..aaba4e1e --- /dev/null +++ b/internal/agent/weights/set.go @@ -0,0 +1,129 @@ +package weights + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" +) + +// IsSet reports whether a source names a set of files rather than one. +// +// By the trailing slash, which is the whole signal: "…/v1/model.gguf" is a file +// and "…/v1/" is the version it belongs to. A flag beside the source would be a +// second place for the two to disagree, and the URL already distinguishes a +// directory from a file everywhere else on the web. +func IsSet(source string) bool { + return isRemote(source) && strings.HasSuffix(source, "/") +} + +// manifest is what the mirror answers for a version. +type manifest struct { + Model string `json:"model"` + Version string `json:"version"` + Files []struct { + Name string `json:"name"` + Bytes int64 `json:"bytes"` + } `json:"files"` + Bytes int64 `json:"bytes"` +} + +// ensureSet downloads every file of a version into one directory. +// +// A directory, because that is what a multi-file model is: vLLM is handed the +// folder and reads the shards and the config out of it, and llama.cpp is handed +// the first part and finds the rest beside it. Downloading them into one place +// is what makes either possible, and a catalog entry naming one file could +// express neither. +// +// The whole set or nothing. A directory with three of four shards in it loads +// as a broken model rather than failing to load, so a partial set is left +// unfinished under its temporary name and the next attempt starts again. +func (f *Fetcher) ensureSet(ctx context.Context, source string) (string, error) { + dir, err := f.cachePath(strings.TrimSuffix(source, "/")) + if err != nil { + return "", err + } + // A complete set is marked by a file written last. Checking for the + // directory alone would treat an interrupted download as finished, which is + // exactly the failure the marker exists to prevent. + done := filepath.Join(dir, ".complete") + if _, err := os.Stat(done); err == nil { + return dir, nil + } + + m, err := f.manifestFor(ctx, source) + if err != nil { + return "", err + } + if len(m.Files) == 0 { + return "", fmt.Errorf("%s holds no files", source) + } + + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + for _, file := range m.Files { + // Rejected rather than joined: a name with a separator in it would + // write outside the directory the mirror said it was in, and the + // mirror is a place an operator copies files into by hand. + if strings.ContainsAny(file.Name, `/\`) || file.Name == ".." { + return "", fmt.Errorf("%s: the mirror offered a file named %q", source, file.Name) + } + target := filepath.Join(dir, file.Name) + if st, err := os.Stat(target); err == nil && st.Size() == file.Bytes { + continue + } + // Reported under the set's own address, because that is what the + // shards are waiting on. Reusing fetch rather than writing a second + // downloader keeps resume, the .part rename and the progress callback + // in one place. + if _, err := f.fetchAs(ctx, source+file.Name, source, target); err != nil { + return "", err + } + } + + if err := os.WriteFile(done, []byte(source+"\n"), 0o644); err != nil { + return "", err + } + return dir, nil +} + +// manifestFor asks the mirror what a version consists of. +func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil) + if err != nil { + return manifest{}, err + } + client := f.Client + if client == nil { + client = &http.Client{} + } + resp, err := client.Do(req) + if err != nil { + return manifest{}, fmt.Errorf("asking %s what it holds: %w", source, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return manifest{}, fmt.Errorf("%s answered %d", source, resp.StatusCode) + } + + var m manifest + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return manifest{}, err + } + if err := json.Unmarshal(body, &m); err != nil { + // The likely cause is a plain file server with a directory index, which + // answers HTML. Said plainly, because "invalid character '<'" sends + // somebody looking at the wrong thing. + return manifest{}, fmt.Errorf( + "%s did not answer with a manifest; a source ending in / has to be a "+ + "mirror version, not a directory listing", source) + } + return m, nil +} diff --git a/internal/agent/weights/set_test.go b/internal/agent/weights/set_test.go new file mode 100644 index 00000000..4a1e167b --- /dev/null +++ b/internal/agent/weights/set_test.go @@ -0,0 +1,167 @@ +package weights + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +// mirrorOf serves a manifest at "/m/v/" and the files beneath it. +func mirrorOf(t *testing.T, files map[string]string) (*Fetcher, *httptest.Server, *int) { + t.Helper() + hits := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/") { + var b strings.Builder + b.WriteString(`{"model":"m","version":"v","files":[`) + first := true + total := 0 + for name, body := range files { + if !first { + b.WriteString(",") + } + first = false + fmt.Fprintf(&b, `{"name":%q,"bytes":%d}`, name, len(body)) + total += len(body) + } + fmt.Fprintf(&b, `],"bytes":%d}`, total) + _, _ = w.Write([]byte(b.String())) + return + } + body, ok := files[strings.TrimPrefix(r.URL.Path, "/m/v/")] + if !ok { + http.NotFound(w, r) + return + } + hits++ + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return &Fetcher{CacheDir: t.TempDir()}, srv, &hits +} + +func TestASetIsFetchedIntoOneDirectory(t *testing.T) { + f, srv, _ := mirrorOf(t, map[string]string{ + "model-00001-of-00002.safetensors": "aaaa", + "model-00002-of-00002.safetensors": "bb", + "config.json": "{}", + }) + + got, err := f.Ensure(t.Context(), srv.URL+"/m/v/") + if err != nil { + t.Fatal(err) + } + st, err := os.Stat(got) + if err != nil || !st.IsDir() { + t.Fatalf("expected a directory, got %q: %v", got, err) + } + // A directory, because that is what a multi-file model is: vLLM reads the + // shards and the config out of the folder, and a catalog entry naming one + // file could express neither. + for _, name := range []string{"model-00001-of-00002.safetensors", "config.json"} { + if _, err := os.Stat(filepath.Join(got, name)); err != nil { + t.Errorf("%s did not arrive: %v", name, err) + } + } +} + +func TestAFinishedSetIsNotFetchedAgain(t *testing.T) { + f, srv, hits := mirrorOf(t, map[string]string{"a.gguf": "aaaa", "b.gguf": "bb"}) + + if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil { + t.Fatal(err) + } + after := *hits + if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil { + t.Fatal(err) + } + if *hits != after { + t.Errorf("the second call downloaded again: %d then %d", after, *hits) + } +} + +func TestAHalfFetchedSetIsNotTreatedAsFinished(t *testing.T) { + f, srv, _ := mirrorOf(t, map[string]string{"a.gguf": "aaaa", "b.gguf": "bb"}) + dir, err := f.cachePath(srv.URL + "/m/v") + if err != nil { + t.Fatal(err) + } + // One file present, no completion marker — an interrupted download. + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "a.gguf"), []byte("aaaa"), 0o644); err != nil { + t.Fatal(err) + } + + got, err := f.Ensure(t.Context(), srv.URL+"/m/v/") + if err != nil { + t.Fatal(err) + } + // A directory with three of four shards loads as a broken model rather + // than failing to load, so the marker is what says the set is whole. + if _, err := os.Stat(filepath.Join(got, "b.gguf")); err != nil { + t.Errorf("the missing file was not fetched: %v", err) + } +} + +func TestAFileAlreadyThereAtTheRightSizeIsNotRefetched(t *testing.T) { + f, srv, hits := mirrorOf(t, map[string]string{"a.gguf": "aaaa", "b.gguf": "bb"}) + dir, _ := f.cachePath(srv.URL + "/m/v") + _ = os.MkdirAll(dir, 0o755) + _ = os.WriteFile(filepath.Join(dir, "a.gguf"), []byte("aaaa"), 0o644) + + if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil { + t.Fatal(err) + } + // Resuming a set means not paying for the shards that already landed. + if *hits != 1 { + t.Errorf("expected only the missing file to be fetched, got %d downloads", *hits) + } +} + +func TestAFilenameThatEscapesTheDirectoryIsRefused(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"files":[{"name":"../../etc/passwd","bytes":1}]}`)) + })) + defer srv.Close() + f := &Fetcher{CacheDir: t.TempDir()} + + _, err := f.Ensure(t.Context(), srv.URL+"/m/v/") + // The mirror is a place an operator copies files into by hand, so what it + // reports is not automatically safe to join onto a path. + if err == nil || !strings.Contains(err.Error(), "offered a file named") { + t.Errorf("expected the name to be refused, got %v", err) + } +} + +func TestADirectoryListingIsNotAManifest(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("
a.gguf")) + })) + defer srv.Close() + f := &Fetcher{CacheDir: t.TempDir()} + + _, err := f.Ensure(t.Context(), srv.URL+"/m/v/") + // "invalid character '<'" sends somebody looking at the wrong thing. + if err == nil || !strings.Contains(err.Error(), "has to be a mirror version") { + t.Errorf("the reason should name the cause, got %v", err) + } +} + +func TestOnlyATrailingSlashMeansASet(t *testing.T) { + for src, want := range map[string]bool{ + "http://mirror/m/v/": true, + "http://mirror/m/v/model.gguf": false, + "/var/lib/models/": false, // a local directory is not fetched + "Qwen/Qwen2.5-7B-Instruct": false, // a repo id, resolved by the engine + } { + if got := IsSet(src); got != want { + t.Errorf("IsSet(%q) = %v, want %v", src, got, want) + } + } +} diff --git a/internal/cli/mirror.go b/internal/cli/mirror.go index 84be076c..934f633d 100644 --- a/internal/cli/mirror.go +++ b/internal/cli/mirror.go @@ -1,6 +1,7 @@ package cli import ( + "encoding/json" "fmt" "log/slog" "net/http" @@ -264,7 +265,7 @@ func newMirrorServeCmd(dir *string) *cobra.Command { // by me. files := &http.Server{ Addr: listen, - Handler: readOnly(http.FileServer(http.Dir(*dir))), + Handler: readOnly(withManifest(*dir, http.FileServer(http.Dir(*dir)))), ReadHeaderTimeout: 10 * time.Second, } admin := &http.Server{ @@ -353,3 +354,47 @@ func catalogSources(cmd *cobra.Command, server string) ([]string, error) { } return out, nil } + +// withManifest answers a request for a version with the list of files in it. +// +// A directory otherwise gets FileServer's HTML index, and something that has to +// know what a version consists of would be left parsing it. A model is +// routinely several files — sharded safetensors, or a GGUF split across parts — +// and a catalog entry names one source, so without a way to ask "what is in +// this version" a sharded model can be mirrored and never served. +// +// On the byte listener rather than the management API on purpose. Fetching +// weights is the one thing every node does, and making it depend on the +// administrative port would undo the split that keeps that port closeable. +func withManifest(root string, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + model, version, ok := versionPath(r.URL.Path) + if !ok { + next.ServeHTTP(w, r) + return + } + m, err := mirror.ManifestFor(root, model, version) + if err != nil { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(m) + }) +} + +// versionPath matches "/model/version/" and nothing else. +// +// The trailing slash is the whole signal: "/m/v/file.gguf" is a file and +// "/m/v/" is the set it belongs to. Requiring it means a source URL says which +// of the two it means without a flag beside it. +func versionPath(p string) (model, version string, ok bool) { + if !strings.HasSuffix(p, "/") { + return "", "", false + } + parts := strings.Split(strings.Trim(p, "/"), "/") + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", false + } + return parts[0], parts[1], true +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index cf8f1fb8..75b2d198 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -172,3 +172,47 @@ func sha256File(path string) (string, error) { } return hex.EncodeToString(h.Sum(nil)), nil } + +// Manifest is what one version of a model consists of. +// +// A version is a directory and a model is routinely several files — sharded +// safetensors, or a GGUF split across parts. A catalog entry names one source, +// so without something that says "these files together are the model" a sharded +// model can be mirrored and never served. +type Manifest struct { + Model string `json:"model"` + Version string `json:"version"` + Files []ManifestFile `json:"files"` + Bytes int64 `json:"bytes"` +} + +// ManifestFile is one file of a version. +type ManifestFile struct { + Name string `json:"name"` + Bytes int64 `json:"bytes"` +} + +// ManifestFor lists the files of one model version. +// +// Built from the directory rather than from a file written alongside it. A +// manifest on disk is a second record of what is there, and the two disagree +// the first time somebody copies a file in by hand — which is a thing the +// mirror is explicitly meant to allow. +func ManifestFor(root, model, version string) (Manifest, error) { + entries, err := List(root) + if err != nil { + return Manifest{}, err + } + m := Manifest{Model: model, Version: version} + for _, e := range entries { + if e.Model != model || e.Version != version { + continue + } + m.Files = append(m.Files, ManifestFile{Name: e.File, Bytes: e.Bytes}) + m.Bytes += e.Bytes + } + if len(m.Files) == 0 { + return Manifest{}, fmt.Errorf("%s/%s: %w", model, version, os.ErrNotExist) + } + return m, nil +} diff --git a/internal/web/admin_serve.go b/internal/web/admin_serve.go index ea7836d3..0a910769 100644 --- a/internal/web/admin_serve.go +++ b/internal/web/admin_serve.go @@ -32,6 +32,8 @@ type serveForm struct { // be written later. Named for the hardware rather than the model, because // a profile is about where a model runs. Profile string + // Files is how many files the version holds, above one for a set. + Files int // MinMemoryGB is the host RAM to ask for, estimated from the weights. // // Filled in rather than left blank. Blank writes no constraint at all, and @@ -63,16 +65,26 @@ func planServe(st mirror.Status, files []mirror.Status, base string, Profile: "cpu", } - // A catalogue entry names one source. A version made of several files has - // no single one, so an entry for it would be an entry for one shard — which - // loads as a broken model rather than failing to load. + // A version of several files is served as the version, not as one of them. + // The source is the directory — a trailing slash — and the agent fetches + // every file in it and hands the engine the folder. if len(files) > 1 { - f.Refusal = fmt.Sprintf( - "this version is %d files, and a catalogue entry names one source. "+ - "Serve a single-file build (a GGUF), or give the engine the "+ - "repository id so it fetches the shards itself", - len(files)) - return f + f.Source = strings.TrimSuffix( + mirror.Entry{Model: st.Model, Version: st.Version}.URL(base), "/") + "/" + f.Files = len(files) + // From whichever file says, because a version is a mixture only when + // somebody has put one there by hand. The weights decide the engine. + for _, one := range files { + if e := engineFor(one.File); e != "" { + f.Engine = e + break + } + } + var total int64 + for _, one := range files { + total += one.BytesDone + } + st.BytesTotal = total } switch f.Engine { diff --git a/internal/web/admin_serve_test.go b/internal/web/admin_serve_test.go index e0fb9475..a048b49f 100644 --- a/internal/web/admin_serve_test.go +++ b/internal/web/admin_serve_test.go @@ -76,22 +76,40 @@ func TestAGGUFCanBeServedStraightFromTheList(t *testing.T) { } } -func TestAShardedVersionIsRefusedWithTheReason(t *testing.T) { +func TestAShardedVersionIsServedAsTheWholeVersion(t *testing.T) { models := []mirror.Status{ held("llama", "v1", "model-00001-of-00002.safetensors"), held("llama", "v1", "model-00002-of-00002.safetensors"), } - body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8), + body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8, "vllm"), "/admin/mirror?serve=llama:v1").Body.String() - // A catalogue entry names one source, so an entry for a sharded version is - // an entry for one shard — which loads as a broken model rather than - // failing to load. - if !strings.Contains(body, "a catalogue entry names one source") { - t.Errorf("the refusal should say why; got %s", body) + // A catalogue entry names one source, and for a version of several files + // that source is the version. Refusing it meant a sharded model could be + // mirrored and never served — which is most of what is not a GGUF. + if !strings.Contains(body, `value="http://mirror:8081/llama/v1/"`) { + t.Errorf("the source should be the version directory; got %s", body) } - if strings.Contains(body, `action="/admin/mirror/serve"`) { - t.Error("no form should be offered for something that cannot be served") + if !strings.Contains(body, "The whole version, 2 files") { + t.Error("the form should say it is fetching a set") + } + if !strings.Contains(body, `action="/admin/mirror/serve"`) { + t.Error("it should be servable") + } +} + +func TestTheEngineForASetComesFromItsWeights(t *testing.T) { + // A version is a mixture only when somebody has put a file there by hand. + // config.json decides nothing; the safetensors do. + models := []mirror.Status{ + held("llama", "v1", "config.json"), + held("llama", "v1", "model-00001-of-00002.safetensors"), + } + body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8, "vllm"), + "/admin/mirror?serve=llama:v1").Body.String() + + if !strings.Contains(body, `name="engine" value="vllm"`) { + t.Errorf("the weights should pick the engine; got %s", body) } } diff --git a/internal/web/templates/admin_mirror.html b/internal/web/templates/admin_mirror.html index 0d4fbbce..f6cac445 100644 --- a/internal/web/templates/admin_mirror.html +++ b/internal/web/templates/admin_mirror.html @@ -93,7 +93,9 @@ reachable from this machine. -
Serving {{.Source}} with {{.Engine}}.
Serving {{.Source}} with {{.Engine}}.
+ {{if .Files}}The whole version, {{.Files}} files: the node fetches them into one
+ directory and the engine is given the directory.{{end}}