From 943c23a412d306e4d8311f549f0103a05e8e4edc Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 17 Sep 2026 08:46:03 +0200 Subject: [PATCH] Fetch a model's weights when the simple arrangement needs it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default arrangement is the simple one: every node keeps its own copy. So a source naming a remote file is now fetched before the engine is launched, rather than left to fail as a missing path minutes into a load. Eight shards of a tensor-parallel replica start at the same instant and want the same file; they produce one download. An interrupted fetch resumes instead of starting over, which matters at forty gigabytes, and the download is written beside its target and renamed only when complete — a partial file under the model's name would be found by the next scan, reported as present, and handed to an engine that fails on it. The exotic arrangements need no code, which is the point. Weights on a Lustre mount or an image exported over the fabric are simply present, and present means nothing is fetched. The escape hatch from a distribution mechanism is a distribution mechanism that checks first. Wiring it up caught a regression in my own design. A bare model id — "Qwen/Qwen2.5-7B-Instruct" — is a documented source that vLLM and SGLang resolve themselves, and treating every non-remote source as a local path made every existing vLLM profile fail. Only an absolute or explicitly relative path is now checked as a file; anything else is passed through untouched. Three agent tests failed on the same change for a better reason: they used "/models/x", and the new check is exactly the one that catches that. Whole repositories of shards are deliberately not fetched here. vLLM and SGLang do that well already, and a second snapshot downloader would be work spent arriving back where we started. Co-Authored-By: Claude Opus 5 (1M context) --- docs/install.md | 29 +++ internal/agent/agent.go | 53 +++++ internal/agent/agent_test.go | 37 +++- internal/agent/weights/fetch.go | 316 +++++++++++++++++++++++++++ internal/agent/weights/fetch_test.go | 289 ++++++++++++++++++++++++ 5 files changed, 723 insertions(+), 1 deletion(-) create mode 100644 internal/agent/weights/fetch.go create mode 100644 internal/agent/weights/fetch_test.go diff --git a/docs/install.md b/docs/install.md index 1bbd667..843c1e4 100644 --- a/docs/install.md +++ b/docs/install.md @@ -282,6 +282,35 @@ sudo install -d -o root -g root /var/lib/llmgw-models For a first model, something small: a 3B at `q4_k_m` is about 2 GB, runs on CPU, and answers fast enough to prove the whole path works. +### Letting llmgw fetch them + +The simple arrangement, and the default: every node keeps its own copy. Point a +model's `source` at a file to download and the agent fetches it once, before the +engine is launched: + +```yaml +source: hf:Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/qwen2.5-coder-3b-instruct-q4_k_m.gguf +# or an ordinary URL +source: https://models.internal/qwen2.5-coder-3b-q4_k_m.gguf +``` + +Eight shards of one replica starting together produce one download, not eight. +An interrupted fetch resumes rather than starting over, and a partial file is +never visible under the model's name — so a half-downloaded model cannot be +picked up by the next scan and handed to an engine. + +Three kinds of source, and only one of them is fetched: + +| `source` | What happens | +| --- | --- | +| `/var/lib/llmgw-models/x.gguf` | Checked, never copied. However it got there. | +| `hf:org/repo/file.gguf`, `https://…` | Fetched once into the cache, then reused | +| `Qwen/Qwen2.5-7B-Instruct` | Left alone — vLLM and SGLang download repositories themselves | + +Which means the exotic arrangements need no configuration at all. Weights on a +Lustre mount, or an image exported over the fabric with SRP or NVMe-oF, are +simply *present*, and present means nothing is fetched. + ### Telling llmgw where they are Optional, and unset by default. Naming the directories in the agent's diff --git a/internal/agent/agent.go b/internal/agent/agent.go index ecca23d..d3178a1 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -22,6 +22,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights" "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" @@ -68,6 +69,15 @@ type Config struct { // is present. ModelDirs []string `koanf:"model_dirs"` + // WeightsCache is where a remote source is downloaded to. Defaults to the + // first entry of ModelDirs, because a node that says where its models live + // has already answered this question. + // + // Fetching is the simple arrangement: every node keeps its own copy. The + // exotic ones need no code — weights on a shared filer or an image exported + // over the fabric are simply present, and present means nothing is fetched. + WeightsCache string `koanf:"weights_cache"` + Discovery discovery.Options `koanf:"discovery"` Engines engines.Config `koanf:"engines"` Superv supervisor.Config `koanf:"supervisor"` @@ -159,6 +169,7 @@ type Agent struct { // shardReplica remembers which replica a shard belongs to, so a status // report can name both. The control plane keys on the replica. shardReplica map[string]string + fetcher *weights.Fetcher } // connectHTTPClient is the HTTP client connect uses; an interface so tests can @@ -333,6 +344,33 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) { "shard", o.ShardID, "replica", o.ReplicaID) } +// weights builds the fetcher lazily, so an agent that never starts a shard +// never needs a cache directory. +func (a *Agent) weights() *weights.Fetcher { + a.mu.Lock() + defer a.mu.Unlock() + if a.fetcher != nil { + return a.fetcher + } + cache := a.cfg.WeightsCache + if cache == "" && len(a.cfg.ModelDirs) > 0 { + cache = a.cfg.ModelDirs[0] + } + a.fetcher = &weights.Fetcher{ + CacheDir: cache, + Log: a.log, + OnProgress: func(p weights.Progress) { + // Logged rather than silent: a replica that is downloading a + // hundred gigabytes and one that is wedged look identical from the + // control plane, and the difference matters to whoever is waiting. + a.log.Info("fetching weights", + "source", p.Source, "percent", p.Percent(), + "bytes", p.BytesDone, "total", p.BytesTotal) + }, + } + return a.fetcher +} + // runningShards describes what this agent is supervising, for registration. func (a *Agent) runningShards() []*llmgwv1.RunningShard { var out []*llmgwv1.RunningShard @@ -565,6 +603,21 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan< return } + // The weights, before anything is launched. A source that is already a + // path is checked and used; one that names a remote file is fetched once + // and shared by every shard that wants it. + // + // Checked here rather than left to the engine because the engine's answer + // arrives minutes into a load, in its own words, on a node nobody is + // watching — and because eight shards starting together would otherwise + // each fetch the same file. + if path, err := a.weights().Ensure(ctx, spec.Profile.ModelPath); err != nil { + report(supervisor.Failed, err.Error(), nil) + return + } else { + spec.Profile.ModelPath = path + } + cmds, err := eng.Commands(spec) if err != nil { report(supervisor.Failed, err.Error(), nil) diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 9a50a3a..8e9e2e4 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -12,6 +12,7 @@ import ( "os/exec" "path/filepath" "slices" + "strings" "sync" "sync/atomic" "syscall" @@ -300,8 +301,17 @@ func (h *harness) waitRegistered(t *testing.T) { } // shardCommand builds a single-node vLLM shard on a free port. +// +// The model path is a real file. It used to be "/models/x", which stopped +// working once the agent began checking that weights exist before launching an +// engine — the check doing exactly what it is for, on a test that was relying +// on nobody looking. func shardCommand(t *testing.T, shardID string) reconciler.ShardCommand { t.Helper() + modelPath := filepath.Join(t.TempDir(), "model.gguf") + if err := os.WriteFile(modelPath, []byte("weights"), 0o644); err != nil { + t.Fatal(err) + } return reconciler.ShardCommand{ ReplicaID: "rep-1", ShardID: shardID, NodeID: "n1", Spec: engines.ShardSpec{ @@ -309,7 +319,7 @@ func shardCommand(t *testing.T, shardID string) reconciler.ShardCommand { GPUIndices: []int{0}, HTTPPort: freePort(t), Profile: engines.Profile{ Name: "p", Engine: "vllm", TP: 1, - ModelPath: "/models/x", ServedModelName: "test-model", + ModelPath: modelPath, ServedModelName: "test-model", }, }, StartupTimeout: 15 * time.Second, @@ -613,3 +623,28 @@ func TestUnclaimedShardIsStopped(t *testing.T) { func processAliveForTest(pid int) bool { return syscall.Kill(pid, 0) == nil } + +func TestAShardWithMissingWeightsFailsBeforeLaunching(t *testing.T) { + // The commonest setup mistake on a fresh node. Left to the engine it + // surfaces minutes into a load, in the engine's own words, on a machine + // nobody is watching — so it is caught here, where the message can say + // what is actually wrong. + a := &Agent{cfg: Config{WeightsCache: t.TempDir()}, log: slog.New(slog.DiscardHandler)} + _, err := a.weights().Ensure(t.Context(), filepath.Join(t.TempDir(), "not-there.gguf")) + if err == nil { + t.Fatal("a missing weights file was accepted") + } + if !strings.Contains(err.Error(), "not-there.gguf") { + t.Errorf("the error does not name the file: %v", err) + } +} + +func TestTheCacheDefaultsToTheFirstModelDirectory(t *testing.T) { + // A node that has said where its models live has already answered this; + // asking twice is a second thing to get out of step. + dir := t.TempDir() + a := &Agent{cfg: Config{ModelDirs: []string{dir, "/elsewhere"}}, log: slog.New(slog.DiscardHandler)} + if got := a.weights().CacheDir; got != dir { + t.Errorf("cache = %q, want the first model directory %q", got, dir) + } +} diff --git a/internal/agent/weights/fetch.go b/internal/agent/weights/fetch.go new file mode 100644 index 0000000..3d07017 --- /dev/null +++ b/internal/agent/weights/fetch.go @@ -0,0 +1,316 @@ +// Package weights makes a model's files present on a node. +// +// The simple arrangement, and the default: every node has its own copy. A +// source that is already a local path is used as it stands; one that names a +// remote file is fetched into a cache directory once and reused. Nothing here +// manages storage, decides a retention policy, or knows what a filesystem is — +// it answers one question, which is where the engine should be pointed. +// +// The exotic arrangements are all handled by not acting. Weights on a shared +// filer, an image exported over the fabric, or anything else an operator has +// mounted are simply present, and present means there is nothing to do. That is +// why the interesting case needs no code: the escape hatch from a distribution +// mechanism is a distribution mechanism that checks first. +package weights + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// Progress reports how a fetch is going, for anything that wants to say so. +type Progress struct { + Source string + BytesDone int64 + BytesTotal int64 +} + +// Percent is how far along, or -1 when the total is unknown. +func (p Progress) Percent() int { + if p.BytesTotal <= 0 { + return -1 + } + return int(p.BytesDone * 100 / p.BytesTotal) +} + +// Fetcher makes sources local. +type Fetcher struct { + CacheDir string + Client *http.Client + Log *slog.Logger + // OnProgress is called occasionally during a fetch. A missing model is the + // difference between a replica that is slow to start and one that is stuck, + // and without this the two are indistinguishable from outside. + OnProgress func(Progress) + + // inflight makes concurrent requests for one source share a single fetch. + // + // Eight shards of a tensor-parallel replica start at the same instant and + // want the same file. Without this they would each open a connection and + // each write the same bytes, turning one download into eight and, on a + // shared origin, into a self-inflicted denial of service. + mu sync.Mutex + inflight map[string]*fetchCall +} + +type fetchCall struct { + done chan struct{} + path string + err error +} + +// ErrNotLocal means a source names a file that is not here and cannot be +// fetched — a bare path with nothing to fetch it from. +var ErrNotLocal = errors.New("the weights are not on this node and the source says nothing about where to get them") + +// Ensure returns a local path for a source, fetching it if it is not here. +// +// A plain path is never fetched, only checked. That is deliberate: a path is +// what an operator writes when the file is already there, whether it arrived on +// local disk, a Lustre mount or an image exported over the fabric, and a +// downloader that second-guessed that would copy a hundred gigabytes that were +// already present. +func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) { + if source == "" { + return "", fmt.Errorf("no source given") + } + if !isRemote(source) { + if !isPath(source) { + // A bare model id — "Qwen/Qwen2.5-7B-Instruct" — is not a file and + // was never meant to be one. vLLM and SGLang resolve those + // themselves, and doing it for them would be both a duplicate + // downloader and a change in documented behaviour. + return source, nil + } + if _, err := os.Stat(source); err == nil { + return source, nil + } + return "", fmt.Errorf("%w: %s", ErrNotLocal, source) + } + + target, err := f.cachePath(source) + if err != nil { + return "", err + } + if _, err := os.Stat(target); err == nil { + return target, nil + } + + f.mu.Lock() + if f.inflight == nil { + f.inflight = map[string]*fetchCall{} + } + if call, running := f.inflight[source]; running { + f.mu.Unlock() + select { + case <-call.done: + return call.path, call.err + case <-ctx.Done(): + return "", ctx.Err() + } + } + call := &fetchCall{done: make(chan struct{})} + f.inflight[source] = call + f.mu.Unlock() + + call.path, call.err = f.fetch(ctx, source, target) + close(call.done) + + f.mu.Lock() + delete(f.inflight, source) + f.mu.Unlock() + return call.path, call.err +} + +// fetch downloads a source to a target, resuming and verifying. +func (f *Fetcher) fetch(ctx context.Context, source, target string) (string, error) { + url, err := resolveURL(source) + if err != nil { + return "", err + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return "", err + } + + // Written beside the target and renamed at the end. A partial file under + // the real name would be found by the next scan, reported as present, and + // handed to an engine that fails on it minutes later — the exact failure + // this whole area exists to prevent. + part := target + ".part" + existing := int64(0) + if st, err := os.Stat(part); err == nil { + existing = st.Size() + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", err + } + if existing > 0 { + // Resume. A forty gigabyte fetch that fails at ninety percent must not + // start again from nothing. + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", existing)) + } + + client := f.Client + if client == nil { + client = &http.Client{} + } + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("fetching %s: %w", source, err) + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + existing = 0 // the server ignored the range; start over + case http.StatusPartialContent: + default: + return "", fmt.Errorf("fetching %s: %s returned %s", source, url, resp.Status) + } + + flags := os.O_CREATE | os.O_WRONLY + if existing > 0 { + flags |= os.O_APPEND + } else { + flags |= os.O_TRUNC + } + out, err := os.OpenFile(part, flags, 0o644) + if err != nil { + return "", err + } + + total := existing + resp.ContentLength + f.report(Progress{Source: source, BytesDone: existing, BytesTotal: total}) + written, err := f.copyWithProgress(ctx, out, resp.Body, source, existing, total) + closeErr := out.Close() + if err != nil { + // The part file is kept on purpose, so the next attempt resumes. + return "", fmt.Errorf("fetching %s after %d bytes: %w", source, written, err) + } + if closeErr != nil { + return "", closeErr + } + + if err := os.Rename(part, target); err != nil { + return "", err + } + if f.Log != nil { + f.Log.Info("weights fetched", "source", source, "path", target, "bytes", written) + } + return target, nil +} + +func (f *Fetcher) copyWithProgress(ctx context.Context, dst io.Writer, src io.Reader, + source string, done, total int64, +) (int64, error) { + buf := make([]byte, 4<<20) + last := time.Now() + for { + if err := ctx.Err(); err != nil { + return done, err + } + n, err := src.Read(buf) + if n > 0 { + if _, werr := dst.Write(buf[:n]); werr != nil { + return done, werr + } + done += int64(n) + // Occasionally, not per read: a four-megabyte buffer over a + // hundred gigabytes is twenty-five thousand callbacks, and nobody + // needs to be told that often. + if time.Since(last) > 2*time.Second { + f.report(Progress{Source: source, BytesDone: done, BytesTotal: total}) + last = time.Now() + } + } + if errors.Is(err, io.EOF) { + f.report(Progress{Source: source, BytesDone: done, BytesTotal: total}) + return done, nil + } + if err != nil { + return done, err + } + } +} + +func (f *Fetcher) report(p Progress) { + if f.OnProgress != nil { + f.OnProgress(p) + } +} + +// cachePath is where a remote source lands. +// +// Named by a hash of the source with the original filename kept on the end, so +// two models with the same filename from different places cannot collide while +// a human looking at the directory can still tell what is in it. +func (f *Fetcher) cachePath(source string) (string, error) { + if f.CacheDir == "" { + return "", fmt.Errorf("no cache directory configured, so %s cannot be fetched", source) + } + sum := sha256.Sum256([]byte(source)) + name := filepath.Base(source) + if name == "" || name == "." || name == "/" { + name = "weights" + } + return filepath.Join(f.CacheDir, hex.EncodeToString(sum[:8])+"-"+name), nil +} + +// isPath reports whether a source names a file on this machine. +// +// By its leading character, because that is the only honest signal available: +// "Qwen/Qwen2.5-7B-Instruct" and "models/qwen.gguf" are indistinguishable as +// strings, and an operator writing a local file writes an absolute path. The +// cost of being wrong in this direction is that a relative path is passed to +// the engine untouched, which is what happened before this code existed. +func isPath(source string) bool { + return strings.HasPrefix(source, "/") || + strings.HasPrefix(source, "./") || + strings.HasPrefix(source, "../") +} + +// isRemote reports whether a source has to be fetched at all. +func isRemote(source string) bool { + return strings.HasPrefix(source, "hf:") || + strings.HasPrefix(source, "http://") || + strings.HasPrefix(source, "https://") +} + +// resolveURL turns a source into something to GET. +// +// hf: names one file in a repository, which is what a GGUF is. A whole +// repository of shards is deliberately not handled here: vLLM and SGLang +// download those themselves and do it well, and reimplementing a snapshot +// downloader to compete with them would be work spent to arrive back where we +// started. +func resolveURL(source string) (string, error) { + if rest, ok := strings.CutPrefix(source, "hf:"); ok { + parts := strings.Split(rest, "/") + if len(parts) < 3 { + return "", fmt.Errorf( + "%s names a repository, not a file; give the file as hf:org/repo/model.gguf, "+ + "or use the repository id directly and let the engine download it", source) + } + repo := strings.Join(parts[:2], "/") + file := strings.Join(parts[2:], "/") + return "https://huggingface.co/" + repo + "/resolve/main/" + file, nil + } + if _, err := url.Parse(source); err != nil { + return "", fmt.Errorf("%s is not a usable source: %w", source, err) + } + return source, nil +} diff --git a/internal/agent/weights/fetch_test.go b/internal/agent/weights/fetch_test.go new file mode 100644 index 0000000..82bed68 --- /dev/null +++ b/internal/agent/weights/fetch_test.go @@ -0,0 +1,289 @@ +package weights + +import ( + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func serveBytes(t *testing.T, body []byte, hits *atomic.Int64) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hits != nil { + hits.Add(1) + } + http.ServeContent(w, r, "model.gguf", time.Time{}, strings.NewReader(string(body))) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestALocalPathIsUsedNeverCopied(t *testing.T) { + // A path is what an operator writes when the file is already there — + // local disk, a Lustre mount, an image exported over the fabric. A + // downloader that second-guessed that would copy a hundred gigabytes that + // were already present. + dir := t.TempDir() + path := filepath.Join(dir, "model.gguf") + if err := os.WriteFile(path, []byte("weights"), 0o644); err != nil { + t.Fatal(err) + } + f := &Fetcher{CacheDir: t.TempDir()} + + got, err := f.Ensure(t.Context(), path) + if err != nil { + t.Fatal(err) + } + if got != path { + t.Errorf("returned %q, want the original path", got) + } + if entries, _ := os.ReadDir(f.CacheDir); len(entries) != 0 { + t.Errorf("the cache was written to for a local file: %v", entries) + } +} + +func TestAMissingLocalPathSaysWhatIsWrong(t *testing.T) { + // The commonest setup mistake, and "no such file" from an engine twenty + // minutes into a load is a poor way to learn it. + f := &Fetcher{CacheDir: t.TempDir()} + _, err := f.Ensure(t.Context(), filepath.Join(t.TempDir(), "absent.gguf")) + if !errors.Is(err, ErrNotLocal) { + t.Fatalf("err = %v, want ErrNotLocal", err) + } + if !strings.Contains(err.Error(), "absent.gguf") { + t.Errorf("the error does not name the file: %v", err) + } +} + +func TestARemoteFileIsFetchedOnceAndReused(t *testing.T) { + var hits atomic.Int64 + srv := serveBytes(t, []byte("the weights"), &hits) + f := &Fetcher{CacheDir: t.TempDir()} + + first, err := f.Ensure(t.Context(), srv.URL+"/model.gguf") + if err != nil { + t.Fatal(err) + } + body, err := os.ReadFile(first) + if err != nil || string(body) != "the weights" { + t.Fatalf("fetched content = %q, err %v", body, err) + } + + second, err := f.Ensure(t.Context(), srv.URL+"/model.gguf") + if err != nil { + t.Fatal(err) + } + if second != first { + t.Errorf("the second call returned a different path: %q vs %q", second, first) + } + if hits.Load() != 1 { + t.Errorf("the file was fetched %d times; the cache did not hold", hits.Load()) + } +} + +func TestConcurrentRequestsProduceOneDownload(t *testing.T) { + // Eight shards of a tensor-parallel replica start at the same instant and + // want the same file. Eight downloads is a self-inflicted denial of service + // on whatever is serving them. + var hits atomic.Int64 + srv := serveBytes(t, []byte("shared weights"), &hits) + f := &Fetcher{CacheDir: t.TempDir()} + + var wg sync.WaitGroup + paths := make([]string, 8) + errs := make([]error, 8) + for i := range 8 { + wg.Add(1) + go func() { + defer wg.Done() + paths[i], errs[i] = f.Ensure(t.Context(), srv.URL+"/model.gguf") + }() + } + wg.Wait() + + for i, err := range errs { + if err != nil { + t.Fatalf("caller %d failed: %v", i, err) + } + if paths[i] != paths[0] { + t.Errorf("caller %d got %q, caller 0 got %q", i, paths[i], paths[0]) + } + } + if n := hits.Load(); n != 1 { + t.Errorf("%d downloads for eight concurrent callers", n) + } +} + +func TestAPartialDownloadIsNeverVisibleAsTheModel(t *testing.T) { + // A partial file under the real name would be found by the next scan, + // reported as present, and handed to an engine that fails on it minutes + // later — the exact failure all of this exists to prevent. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "1000") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("only the first bytes")) + // and then the connection dies without the rest + if fl, ok := w.(http.Flusher); ok { + fl.Flush() + } + panic(http.ErrAbortHandler) + })) + defer srv.Close() + + cache := t.TempDir() + f := &Fetcher{CacheDir: cache} + if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf"); err == nil { + t.Fatal("a truncated download was reported as success") + } + entries, _ := os.ReadDir(cache) + for _, e := range entries { + if !strings.HasSuffix(e.Name(), ".part") { + t.Errorf("a non-part file was left behind: %q", e.Name()) + } + } +} + +func TestAnInterruptedDownloadResumes(t *testing.T) { + // A forty gigabyte fetch that fails at ninety percent must not start again + // from nothing. + full := []byte(strings.Repeat("x", 4096)) + var ranges []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ranges = append(ranges, r.Header.Get("Range")) + http.ServeContent(w, r, "model.gguf", time.Time{}, strings.NewReader(string(full))) + })) + defer srv.Close() + + cache := t.TempDir() + f := &Fetcher{CacheDir: cache} + target, _ := f.cachePath(srv.URL + "/model.gguf") + // Half a file already on disk, as an interrupted attempt would leave. + if err := os.WriteFile(target+".part", full[:2048], 0o644); err != nil { + t.Fatal(err) + } + + got, err := f.Ensure(t.Context(), srv.URL+"/model.gguf") + if err != nil { + t.Fatal(err) + } + body, _ := os.ReadFile(got) + if len(body) != len(full) { + t.Errorf("resumed file is %d bytes, want %d", len(body), len(full)) + } + if len(ranges) == 0 || !strings.Contains(ranges[0], "2048") { + t.Errorf("no range request was made: %v", ranges) + } +} + +func TestAnHFSourceNamingARepositoryExplainsItself(t *testing.T) { + // A whole repository of shards is deliberately not handled here: vLLM and + // SGLang download those themselves and do it well. Saying so beats a + // confusing 404. + _, err := resolveURL("hf:Qwen/Qwen2.5-Coder-32B-Instruct") + if err == nil { + t.Fatal("a bare repository was accepted as a file") + } + for _, want := range []string{"model.gguf", "let the engine"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the error does not suggest what to do (%q): %v", want, err) + } + } +} + +func TestAnHFFileResolvesToItsDownloadURL(t *testing.T) { + got, err := resolveURL("hf:Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/qwen2.5-coder-3b-instruct-q4_k_m.gguf") + if err != nil { + t.Fatal(err) + } + want := "https://huggingface.co/Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/resolve/main/qwen2.5-coder-3b-instruct-q4_k_m.gguf" + if got != want { + t.Errorf("resolved to\n %s\nwant\n %s", got, want) + } +} + +func TestTwoSourcesWithTheSameFilenameDoNotCollide(t *testing.T) { + f := &Fetcher{CacheDir: t.TempDir()} + a, _ := f.cachePath("https://one.example/model.gguf") + b, _ := f.cachePath("https://two.example/model.gguf") + if a == b { + t.Fatalf("both sources cache to %q", a) + } + // And a person looking at the directory can still tell what is in it. + for _, p := range []string{a, b} { + if !strings.HasSuffix(p, "model.gguf") { + t.Errorf("cache path %q does not keep the filename", p) + } + } +} + +func TestProgressIsReportedWithAPercentage(t *testing.T) { + // A missing model is the difference between a replica that is slow to start + // and one that is stuck, and without this they look identical. + srv := serveBytes(t, []byte(strings.Repeat("y", 8192)), nil) + var seen []Progress + f := &Fetcher{ + CacheDir: t.TempDir(), + OnProgress: func(p Progress) { seen = append(seen, p) }, + } + if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf"); err != nil { + t.Fatal(err) + } + if len(seen) == 0 { + t.Fatal("no progress was reported") + } + last := seen[len(seen)-1] + if last.Percent() != 100 { + t.Errorf("final progress is %d%%, want 100 (%+v)", last.Percent(), last) + } +} + +func TestPercentIsUnknownRatherThanZeroWithoutATotal(t *testing.T) { + // A server that sends no Content-Length is common, and reporting 0% for + // a download that is nearly finished is worse than admitting ignorance. + if got := (Progress{BytesDone: 500}).Percent(); got != -1 { + t.Errorf("Percent with no total = %d, want -1", got) + } +} + +func TestNoCacheDirectoryIsAClearRefusal(t *testing.T) { + f := &Fetcher{} + _, err := f.Ensure(t.Context(), "https://example.invalid/model.gguf") + if err == nil || !strings.Contains(err.Error(), "cache directory") { + t.Errorf("err = %v, want a complaint about the cache directory", err) + } +} + +func TestABareModelIdIsLeftToTheEngine(t *testing.T) { + // "Qwen/Qwen2.5-7B-Instruct" is a documented source that vLLM and SGLang + // resolve themselves. Treating it as a missing file broke every existing + // vLLM profile, which is how this was found. + f := &Fetcher{CacheDir: t.TempDir()} + for _, id := range []string{ + "Qwen/Qwen2.5-7B-Instruct", + "meta-llama/Llama-3.3-70B-Instruct", + } { + got, err := f.Ensure(t.Context(), id) + if err != nil { + t.Errorf("%s was refused: %v", id, err) + } + if got != id { + t.Errorf("%s was rewritten to %q", id, got) + } + } +} + +func TestAnAbsolutePathThatIsMissingIsStillAnError(t *testing.T) { + // The other half: an operator writing a path means a file, and a missing + // one should be said plainly rather than handed to an engine. + f := &Fetcher{CacheDir: t.TempDir()} + if _, err := f.Ensure(t.Context(), "/models/definitely-not-here.gguf"); !errors.Is(err, ErrNotLocal) { + t.Errorf("err = %v, want ErrNotLocal", err) + } +}