diff --git a/docs/install.md b/docs/install.md index 1bbd667..843c1e4 100644 --- a/docs/install.md +++ b/docs/install.md @@ -282,6 +282,35 @@ sudo install -d -o root -g root /var/lib/llmgw-models For a first model, something small: a 3B at `q4_k_m` is about 2 GB, runs on CPU, and answers fast enough to prove the whole path works. +### Letting llmgw fetch them + +The simple arrangement, and the default: every node keeps its own copy. Point a +model's `source` at a file to download and the agent fetches it once, before the +engine is launched: + +```yaml +source: hf:Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/qwen2.5-coder-3b-instruct-q4_k_m.gguf +# or an ordinary URL +source: https://models.internal/qwen2.5-coder-3b-q4_k_m.gguf +``` + +Eight shards of one replica starting together produce one download, not eight. +An interrupted fetch resumes rather than starting over, and a partial file is +never visible under the model's name — so a half-downloaded model cannot be +picked up by the next scan and handed to an engine. + +Three kinds of source, and only one of them is fetched: + +| `source` | What happens | +| --- | --- | +| `/var/lib/llmgw-models/x.gguf` | Checked, never copied. However it got there. | +| `hf:org/repo/file.gguf`, `https://…` | Fetched once into the cache, then reused | +| `Qwen/Qwen2.5-7B-Instruct` | Left alone — vLLM and SGLang download repositories themselves | + +Which means the exotic arrangements need no configuration at all. Weights on a +Lustre mount, or an image exported over the fabric with SRP or NVMe-oF, are +simply *present*, and present means nothing is fetched. + ### Telling llmgw where they are Optional, and unset by default. Naming the directories in the agent's diff --git a/internal/agent/agent.go b/internal/agent/agent.go index ecca23d..d3178a1 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -22,6 +22,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights" "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" @@ -68,6 +69,15 @@ type Config struct { // is present. ModelDirs []string `koanf:"model_dirs"` + // WeightsCache is where a remote source is downloaded to. Defaults to the + // first entry of ModelDirs, because a node that says where its models live + // has already answered this question. + // + // Fetching is the simple arrangement: every node keeps its own copy. The + // exotic ones need no code — weights on a shared filer or an image exported + // over the fabric are simply present, and present means nothing is fetched. + WeightsCache string `koanf:"weights_cache"` + Discovery discovery.Options `koanf:"discovery"` Engines engines.Config `koanf:"engines"` Superv supervisor.Config `koanf:"supervisor"` @@ -159,6 +169,7 @@ type Agent struct { // shardReplica remembers which replica a shard belongs to, so a status // report can name both. The control plane keys on the replica. shardReplica map[string]string + fetcher *weights.Fetcher } // connectHTTPClient is the HTTP client connect uses; an interface so tests can @@ -333,6 +344,33 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) { "shard", o.ShardID, "replica", o.ReplicaID) } +// weights builds the fetcher lazily, so an agent that never starts a shard +// never needs a cache directory. +func (a *Agent) weights() *weights.Fetcher { + a.mu.Lock() + defer a.mu.Unlock() + if a.fetcher != nil { + return a.fetcher + } + cache := a.cfg.WeightsCache + if cache == "" && len(a.cfg.ModelDirs) > 0 { + cache = a.cfg.ModelDirs[0] + } + a.fetcher = &weights.Fetcher{ + CacheDir: cache, + Log: a.log, + OnProgress: func(p weights.Progress) { + // Logged rather than silent: a replica that is downloading a + // hundred gigabytes and one that is wedged look identical from the + // control plane, and the difference matters to whoever is waiting. + a.log.Info("fetching weights", + "source", p.Source, "percent", p.Percent(), + "bytes", p.BytesDone, "total", p.BytesTotal) + }, + } + return a.fetcher +} + // runningShards describes what this agent is supervising, for registration. func (a *Agent) runningShards() []*llmgwv1.RunningShard { var out []*llmgwv1.RunningShard @@ -565,6 +603,21 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan< return } + // The weights, before anything is launched. A source that is already a + // path is checked and used; one that names a remote file is fetched once + // and shared by every shard that wants it. + // + // Checked here rather than left to the engine because the engine's answer + // arrives minutes into a load, in its own words, on a node nobody is + // watching — and because eight shards starting together would otherwise + // each fetch the same file. + if path, err := a.weights().Ensure(ctx, spec.Profile.ModelPath); err != nil { + report(supervisor.Failed, err.Error(), nil) + return + } else { + spec.Profile.ModelPath = path + } + cmds, err := eng.Commands(spec) if err != nil { report(supervisor.Failed, err.Error(), nil) diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 9a50a3a..8e9e2e4 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -12,6 +12,7 @@ import ( "os/exec" "path/filepath" "slices" + "strings" "sync" "sync/atomic" "syscall" @@ -300,8 +301,17 @@ func (h *harness) waitRegistered(t *testing.T) { } // shardCommand builds a single-node vLLM shard on a free port. +// +// The model path is a real file. It used to be "/models/x", which stopped +// working once the agent began checking that weights exist before launching an +// engine — the check doing exactly what it is for, on a test that was relying +// on nobody looking. func shardCommand(t *testing.T, shardID string) reconciler.ShardCommand { t.Helper() + modelPath := filepath.Join(t.TempDir(), "model.gguf") + if err := os.WriteFile(modelPath, []byte("weights"), 0o644); err != nil { + t.Fatal(err) + } return reconciler.ShardCommand{ ReplicaID: "rep-1", ShardID: shardID, NodeID: "n1", Spec: engines.ShardSpec{ @@ -309,7 +319,7 @@ func shardCommand(t *testing.T, shardID string) reconciler.ShardCommand { GPUIndices: []int{0}, HTTPPort: freePort(t), Profile: engines.Profile{ Name: "p", Engine: "vllm", TP: 1, - ModelPath: "/models/x", ServedModelName: "test-model", + ModelPath: modelPath, ServedModelName: "test-model", }, }, StartupTimeout: 15 * time.Second, @@ -613,3 +623,28 @@ func TestUnclaimedShardIsStopped(t *testing.T) { func processAliveForTest(pid int) bool { return syscall.Kill(pid, 0) == nil } + +func TestAShardWithMissingWeightsFailsBeforeLaunching(t *testing.T) { + // The commonest setup mistake on a fresh node. Left to the engine it + // surfaces minutes into a load, in the engine's own words, on a machine + // nobody is watching — so it is caught here, where the message can say + // what is actually wrong. + a := &Agent{cfg: Config{WeightsCache: t.TempDir()}, log: slog.New(slog.DiscardHandler)} + _, err := a.weights().Ensure(t.Context(), filepath.Join(t.TempDir(), "not-there.gguf")) + if err == nil { + t.Fatal("a missing weights file was accepted") + } + if !strings.Contains(err.Error(), "not-there.gguf") { + t.Errorf("the error does not name the file: %v", err) + } +} + +func TestTheCacheDefaultsToTheFirstModelDirectory(t *testing.T) { + // A node that has said where its models live has already answered this; + // asking twice is a second thing to get out of step. + dir := t.TempDir() + a := &Agent{cfg: Config{ModelDirs: []string{dir, "/elsewhere"}}, log: slog.New(slog.DiscardHandler)} + if got := a.weights().CacheDir; got != dir { + t.Errorf("cache = %q, want the first model directory %q", got, dir) + } +} diff --git a/internal/agent/weights/fetch.go b/internal/agent/weights/fetch.go new file mode 100644 index 0000000..3d07017 --- /dev/null +++ b/internal/agent/weights/fetch.go @@ -0,0 +1,316 @@ +// Package weights makes a model's files present on a node. +// +// The simple arrangement, and the default: every node has its own copy. A +// source that is already a local path is used as it stands; one that names a +// remote file is fetched into a cache directory once and reused. Nothing here +// manages storage, decides a retention policy, or knows what a filesystem is — +// it answers one question, which is where the engine should be pointed. +// +// The exotic arrangements are all handled by not acting. Weights on a shared +// filer, an image exported over the fabric, or anything else an operator has +// mounted are simply present, and present means there is nothing to do. That is +// why the interesting case needs no code: the escape hatch from a distribution +// mechanism is a distribution mechanism that checks first. +package weights + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// Progress reports how a fetch is going, for anything that wants to say so. +type Progress struct { + Source string + BytesDone int64 + BytesTotal int64 +} + +// Percent is how far along, or -1 when the total is unknown. +func (p Progress) Percent() int { + if p.BytesTotal <= 0 { + return -1 + } + return int(p.BytesDone * 100 / p.BytesTotal) +} + +// Fetcher makes sources local. +type Fetcher struct { + CacheDir string + Client *http.Client + Log *slog.Logger + // OnProgress is called occasionally during a fetch. A missing model is the + // difference between a replica that is slow to start and one that is stuck, + // and without this the two are indistinguishable from outside. + OnProgress func(Progress) + + // inflight makes concurrent requests for one source share a single fetch. + // + // Eight shards of a tensor-parallel replica start at the same instant and + // want the same file. Without this they would each open a connection and + // each write the same bytes, turning one download into eight and, on a + // shared origin, into a self-inflicted denial of service. + mu sync.Mutex + inflight map[string]*fetchCall +} + +type fetchCall struct { + done chan struct{} + path string + err error +} + +// ErrNotLocal means a source names a file that is not here and cannot be +// fetched — a bare path with nothing to fetch it from. +var ErrNotLocal = errors.New("the weights are not on this node and the source says nothing about where to get them") + +// Ensure returns a local path for a source, fetching it if it is not here. +// +// A plain path is never fetched, only checked. That is deliberate: a path is +// what an operator writes when the file is already there, whether it arrived on +// local disk, a Lustre mount or an image exported over the fabric, and a +// downloader that second-guessed that would copy a hundred gigabytes that were +// already present. +func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) { + if source == "" { + return "", fmt.Errorf("no source given") + } + if !isRemote(source) { + if !isPath(source) { + // A bare model id — "Qwen/Qwen2.5-7B-Instruct" — is not a file and + // was never meant to be one. vLLM and SGLang resolve those + // themselves, and doing it for them would be both a duplicate + // downloader and a change in documented behaviour. + return source, nil + } + if _, err := os.Stat(source); err == nil { + return source, nil + } + return "", fmt.Errorf("%w: %s", ErrNotLocal, source) + } + + target, err := f.cachePath(source) + if err != nil { + return "", err + } + if _, err := os.Stat(target); err == nil { + return target, nil + } + + f.mu.Lock() + if f.inflight == nil { + f.inflight = map[string]*fetchCall{} + } + if call, running := f.inflight[source]; running { + f.mu.Unlock() + select { + case <-call.done: + return call.path, call.err + case <-ctx.Done(): + return "", ctx.Err() + } + } + call := &fetchCall{done: make(chan struct{})} + f.inflight[source] = call + f.mu.Unlock() + + call.path, call.err = f.fetch(ctx, source, target) + close(call.done) + + f.mu.Lock() + delete(f.inflight, source) + f.mu.Unlock() + return call.path, call.err +} + +// fetch downloads a source to a target, resuming and verifying. +func (f *Fetcher) fetch(ctx context.Context, source, target string) (string, error) { + url, err := resolveURL(source) + if err != nil { + return "", err + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return "", err + } + + // Written beside the target and renamed at the end. A partial file under + // the real name would be found by the next scan, reported as present, and + // handed to an engine that fails on it minutes later — the exact failure + // this whole area exists to prevent. + part := target + ".part" + existing := int64(0) + if st, err := os.Stat(part); err == nil { + existing = st.Size() + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", err + } + if existing > 0 { + // Resume. A forty gigabyte fetch that fails at ninety percent must not + // start again from nothing. + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", existing)) + } + + client := f.Client + if client == nil { + client = &http.Client{} + } + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("fetching %s: %w", source, err) + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + existing = 0 // the server ignored the range; start over + case http.StatusPartialContent: + default: + return "", fmt.Errorf("fetching %s: %s returned %s", source, url, resp.Status) + } + + flags := os.O_CREATE | os.O_WRONLY + if existing > 0 { + flags |= os.O_APPEND + } else { + flags |= os.O_TRUNC + } + out, err := os.OpenFile(part, flags, 0o644) + if err != nil { + return "", err + } + + total := existing + resp.ContentLength + f.report(Progress{Source: source, BytesDone: existing, BytesTotal: total}) + written, err := f.copyWithProgress(ctx, out, resp.Body, source, existing, total) + closeErr := out.Close() + if err != nil { + // The part file is kept on purpose, so the next attempt resumes. + return "", fmt.Errorf("fetching %s after %d bytes: %w", source, written, err) + } + if closeErr != nil { + return "", closeErr + } + + if err := os.Rename(part, target); err != nil { + return "", err + } + if f.Log != nil { + f.Log.Info("weights fetched", "source", source, "path", target, "bytes", written) + } + return target, nil +} + +func (f *Fetcher) copyWithProgress(ctx context.Context, dst io.Writer, src io.Reader, + source string, done, total int64, +) (int64, error) { + buf := make([]byte, 4<<20) + last := time.Now() + for { + if err := ctx.Err(); err != nil { + return done, err + } + n, err := src.Read(buf) + if n > 0 { + if _, werr := dst.Write(buf[:n]); werr != nil { + return done, werr + } + done += int64(n) + // Occasionally, not per read: a four-megabyte buffer over a + // hundred gigabytes is twenty-five thousand callbacks, and nobody + // needs to be told that often. + if time.Since(last) > 2*time.Second { + f.report(Progress{Source: source, BytesDone: done, BytesTotal: total}) + last = time.Now() + } + } + if errors.Is(err, io.EOF) { + f.report(Progress{Source: source, BytesDone: done, BytesTotal: total}) + return done, nil + } + if err != nil { + return done, err + } + } +} + +func (f *Fetcher) report(p Progress) { + if f.OnProgress != nil { + f.OnProgress(p) + } +} + +// cachePath is where a remote source lands. +// +// Named by a hash of the source with the original filename kept on the end, so +// two models with the same filename from different places cannot collide while +// a human looking at the directory can still tell what is in it. +func (f *Fetcher) cachePath(source string) (string, error) { + if f.CacheDir == "" { + return "", fmt.Errorf("no cache directory configured, so %s cannot be fetched", source) + } + sum := sha256.Sum256([]byte(source)) + name := filepath.Base(source) + if name == "" || name == "." || name == "/" { + name = "weights" + } + return filepath.Join(f.CacheDir, hex.EncodeToString(sum[:8])+"-"+name), nil +} + +// isPath reports whether a source names a file on this machine. +// +// By its leading character, because that is the only honest signal available: +// "Qwen/Qwen2.5-7B-Instruct" and "models/qwen.gguf" are indistinguishable as +// strings, and an operator writing a local file writes an absolute path. The +// cost of being wrong in this direction is that a relative path is passed to +// the engine untouched, which is what happened before this code existed. +func isPath(source string) bool { + return strings.HasPrefix(source, "/") || + strings.HasPrefix(source, "./") || + strings.HasPrefix(source, "../") +} + +// isRemote reports whether a source has to be fetched at all. +func isRemote(source string) bool { + return strings.HasPrefix(source, "hf:") || + strings.HasPrefix(source, "http://") || + strings.HasPrefix(source, "https://") +} + +// resolveURL turns a source into something to GET. +// +// hf: names one file in a repository, which is what a GGUF is. A whole +// repository of shards is deliberately not handled here: vLLM and SGLang +// download those themselves and do it well, and reimplementing a snapshot +// downloader to compete with them would be work spent to arrive back where we +// started. +func resolveURL(source string) (string, error) { + if rest, ok := strings.CutPrefix(source, "hf:"); ok { + parts := strings.Split(rest, "/") + if len(parts) < 3 { + return "", fmt.Errorf( + "%s names a repository, not a file; give the file as hf:org/repo/model.gguf, "+ + "or use the repository id directly and let the engine download it", source) + } + repo := strings.Join(parts[:2], "/") + file := strings.Join(parts[2:], "/") + return "https://huggingface.co/" + repo + "/resolve/main/" + file, nil + } + if _, err := url.Parse(source); err != nil { + return "", fmt.Errorf("%s is not a usable source: %w", source, err) + } + return source, nil +} diff --git a/internal/agent/weights/fetch_test.go b/internal/agent/weights/fetch_test.go new file mode 100644 index 0000000..82bed68 --- /dev/null +++ b/internal/agent/weights/fetch_test.go @@ -0,0 +1,289 @@ +package weights + +import ( + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func serveBytes(t *testing.T, body []byte, hits *atomic.Int64) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hits != nil { + hits.Add(1) + } + http.ServeContent(w, r, "model.gguf", time.Time{}, strings.NewReader(string(body))) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestALocalPathIsUsedNeverCopied(t *testing.T) { + // A path is what an operator writes when the file is already there — + // local disk, a Lustre mount, an image exported over the fabric. A + // downloader that second-guessed that would copy a hundred gigabytes that + // were already present. + dir := t.TempDir() + path := filepath.Join(dir, "model.gguf") + if err := os.WriteFile(path, []byte("weights"), 0o644); err != nil { + t.Fatal(err) + } + f := &Fetcher{CacheDir: t.TempDir()} + + got, err := f.Ensure(t.Context(), path) + if err != nil { + t.Fatal(err) + } + if got != path { + t.Errorf("returned %q, want the original path", got) + } + if entries, _ := os.ReadDir(f.CacheDir); len(entries) != 0 { + t.Errorf("the cache was written to for a local file: %v", entries) + } +} + +func TestAMissingLocalPathSaysWhatIsWrong(t *testing.T) { + // The commonest setup mistake, and "no such file" from an engine twenty + // minutes into a load is a poor way to learn it. + f := &Fetcher{CacheDir: t.TempDir()} + _, err := f.Ensure(t.Context(), filepath.Join(t.TempDir(), "absent.gguf")) + if !errors.Is(err, ErrNotLocal) { + t.Fatalf("err = %v, want ErrNotLocal", err) + } + if !strings.Contains(err.Error(), "absent.gguf") { + t.Errorf("the error does not name the file: %v", err) + } +} + +func TestARemoteFileIsFetchedOnceAndReused(t *testing.T) { + var hits atomic.Int64 + srv := serveBytes(t, []byte("the weights"), &hits) + f := &Fetcher{CacheDir: t.TempDir()} + + first, err := f.Ensure(t.Context(), srv.URL+"/model.gguf") + if err != nil { + t.Fatal(err) + } + body, err := os.ReadFile(first) + if err != nil || string(body) != "the weights" { + t.Fatalf("fetched content = %q, err %v", body, err) + } + + second, err := f.Ensure(t.Context(), srv.URL+"/model.gguf") + if err != nil { + t.Fatal(err) + } + if second != first { + t.Errorf("the second call returned a different path: %q vs %q", second, first) + } + if hits.Load() != 1 { + t.Errorf("the file was fetched %d times; the cache did not hold", hits.Load()) + } +} + +func TestConcurrentRequestsProduceOneDownload(t *testing.T) { + // Eight shards of a tensor-parallel replica start at the same instant and + // want the same file. Eight downloads is a self-inflicted denial of service + // on whatever is serving them. + var hits atomic.Int64 + srv := serveBytes(t, []byte("shared weights"), &hits) + f := &Fetcher{CacheDir: t.TempDir()} + + var wg sync.WaitGroup + paths := make([]string, 8) + errs := make([]error, 8) + for i := range 8 { + wg.Add(1) + go func() { + defer wg.Done() + paths[i], errs[i] = f.Ensure(t.Context(), srv.URL+"/model.gguf") + }() + } + wg.Wait() + + for i, err := range errs { + if err != nil { + t.Fatalf("caller %d failed: %v", i, err) + } + if paths[i] != paths[0] { + t.Errorf("caller %d got %q, caller 0 got %q", i, paths[i], paths[0]) + } + } + if n := hits.Load(); n != 1 { + t.Errorf("%d downloads for eight concurrent callers", n) + } +} + +func TestAPartialDownloadIsNeverVisibleAsTheModel(t *testing.T) { + // A partial file under the real name would be found by the next scan, + // reported as present, and handed to an engine that fails on it minutes + // later — the exact failure all of this exists to prevent. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "1000") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("only the first bytes")) + // and then the connection dies without the rest + if fl, ok := w.(http.Flusher); ok { + fl.Flush() + } + panic(http.ErrAbortHandler) + })) + defer srv.Close() + + cache := t.TempDir() + f := &Fetcher{CacheDir: cache} + if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf"); err == nil { + t.Fatal("a truncated download was reported as success") + } + entries, _ := os.ReadDir(cache) + for _, e := range entries { + if !strings.HasSuffix(e.Name(), ".part") { + t.Errorf("a non-part file was left behind: %q", e.Name()) + } + } +} + +func TestAnInterruptedDownloadResumes(t *testing.T) { + // A forty gigabyte fetch that fails at ninety percent must not start again + // from nothing. + full := []byte(strings.Repeat("x", 4096)) + var ranges []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ranges = append(ranges, r.Header.Get("Range")) + http.ServeContent(w, r, "model.gguf", time.Time{}, strings.NewReader(string(full))) + })) + defer srv.Close() + + cache := t.TempDir() + f := &Fetcher{CacheDir: cache} + target, _ := f.cachePath(srv.URL + "/model.gguf") + // Half a file already on disk, as an interrupted attempt would leave. + if err := os.WriteFile(target+".part", full[:2048], 0o644); err != nil { + t.Fatal(err) + } + + got, err := f.Ensure(t.Context(), srv.URL+"/model.gguf") + if err != nil { + t.Fatal(err) + } + body, _ := os.ReadFile(got) + if len(body) != len(full) { + t.Errorf("resumed file is %d bytes, want %d", len(body), len(full)) + } + if len(ranges) == 0 || !strings.Contains(ranges[0], "2048") { + t.Errorf("no range request was made: %v", ranges) + } +} + +func TestAnHFSourceNamingARepositoryExplainsItself(t *testing.T) { + // A whole repository of shards is deliberately not handled here: vLLM and + // SGLang download those themselves and do it well. Saying so beats a + // confusing 404. + _, err := resolveURL("hf:Qwen/Qwen2.5-Coder-32B-Instruct") + if err == nil { + t.Fatal("a bare repository was accepted as a file") + } + for _, want := range []string{"model.gguf", "let the engine"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the error does not suggest what to do (%q): %v", want, err) + } + } +} + +func TestAnHFFileResolvesToItsDownloadURL(t *testing.T) { + got, err := resolveURL("hf:Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/qwen2.5-coder-3b-instruct-q4_k_m.gguf") + if err != nil { + t.Fatal(err) + } + want := "https://huggingface.co/Qwen/Qwen2.5-Coder-3B-Instruct-GGUF/resolve/main/qwen2.5-coder-3b-instruct-q4_k_m.gguf" + if got != want { + t.Errorf("resolved to\n %s\nwant\n %s", got, want) + } +} + +func TestTwoSourcesWithTheSameFilenameDoNotCollide(t *testing.T) { + f := &Fetcher{CacheDir: t.TempDir()} + a, _ := f.cachePath("https://one.example/model.gguf") + b, _ := f.cachePath("https://two.example/model.gguf") + if a == b { + t.Fatalf("both sources cache to %q", a) + } + // And a person looking at the directory can still tell what is in it. + for _, p := range []string{a, b} { + if !strings.HasSuffix(p, "model.gguf") { + t.Errorf("cache path %q does not keep the filename", p) + } + } +} + +func TestProgressIsReportedWithAPercentage(t *testing.T) { + // A missing model is the difference between a replica that is slow to start + // and one that is stuck, and without this they look identical. + srv := serveBytes(t, []byte(strings.Repeat("y", 8192)), nil) + var seen []Progress + f := &Fetcher{ + CacheDir: t.TempDir(), + OnProgress: func(p Progress) { seen = append(seen, p) }, + } + if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf"); err != nil { + t.Fatal(err) + } + if len(seen) == 0 { + t.Fatal("no progress was reported") + } + last := seen[len(seen)-1] + if last.Percent() != 100 { + t.Errorf("final progress is %d%%, want 100 (%+v)", last.Percent(), last) + } +} + +func TestPercentIsUnknownRatherThanZeroWithoutATotal(t *testing.T) { + // A server that sends no Content-Length is common, and reporting 0% for + // a download that is nearly finished is worse than admitting ignorance. + if got := (Progress{BytesDone: 500}).Percent(); got != -1 { + t.Errorf("Percent with no total = %d, want -1", got) + } +} + +func TestNoCacheDirectoryIsAClearRefusal(t *testing.T) { + f := &Fetcher{} + _, err := f.Ensure(t.Context(), "https://example.invalid/model.gguf") + if err == nil || !strings.Contains(err.Error(), "cache directory") { + t.Errorf("err = %v, want a complaint about the cache directory", err) + } +} + +func TestABareModelIdIsLeftToTheEngine(t *testing.T) { + // "Qwen/Qwen2.5-7B-Instruct" is a documented source that vLLM and SGLang + // resolve themselves. Treating it as a missing file broke every existing + // vLLM profile, which is how this was found. + f := &Fetcher{CacheDir: t.TempDir()} + for _, id := range []string{ + "Qwen/Qwen2.5-7B-Instruct", + "meta-llama/Llama-3.3-70B-Instruct", + } { + got, err := f.Ensure(t.Context(), id) + if err != nil { + t.Errorf("%s was refused: %v", id, err) + } + if got != id { + t.Errorf("%s was rewritten to %q", id, got) + } + } +} + +func TestAnAbsolutePathThatIsMissingIsStillAnError(t *testing.T) { + // The other half: an operator writing a path means a file, and a missing + // one should be said plainly rather than handed to an engine. + f := &Fetcher{CacheDir: t.TempDir()} + if _, err := f.Ensure(t.Context(), "/models/definitely-not-here.gguf"); !errors.Is(err, ErrNotLocal) { + t.Errorf("err = %v, want ErrNotLocal", err) + } +}