Weights that run code are refused
Python pickles (.bin, .pt, .pth, .ckpt, .pkl) run whatever they hold when loaded, as the engine's user, on the node; that is how the malicious models found on Hugging Face work. The node refuses such a source, or a mirror set holding one, before fetching anything. A model added with one, or given one, is refused, while one already in the catalog still loads and is refused by its node with the same words. vLLM and SGLang downloading a model by its id get --load-format safetensors. The mirror page marks the file. allow_pickle: true on a model lets it through on purpose, carried to the node in EngineProfile.allow_pickle. BACKLOG M lists the rest: checksums against Hugging Face on every path, its own scan, trusted publishers, engines without the internet, an eval before an untrusted model serves. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6752938e6d
commit
4466d2d15a
31 files changed
+447
-44
No files matched your search
@@ -1124,7 +1124,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
},
|
||||
})
|
||||
})
|
||||
if path, err := a.weights().Ensure(ctx, spec.Profile.ModelPath); err != nil {
|
||||
if path, err := a.weights().Ensure(ctx, spec.Profile.ModelPath, spec.Profile.AllowPickle); err != nil {
|
||||
stop()
|
||||
report(supervisor.Failed, err.Error(), nil)
|
||||
return
|
||||
@@ -1145,7 +1145,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
send(&llmgwv1.ShardStatus{State: agentsvc.StateToWire(supervisor.Starting),
|
||||
Fetch: &llmgwv1.FetchProgress{Source: p.Source, BytesDone: p.BytesDone, BytesTotal: p.BytesTotal}})
|
||||
})
|
||||
path, err := a.weights().Ensure(ctx, m["source"])
|
||||
path, err := a.weights().Ensure(ctx, m["source"], m["allow_pickle"] == "true")
|
||||
stop()
|
||||
if err != nil {
|
||||
report(supervisor.Failed, fmt.Sprintf("model %s: %v", m["name"], err), nil)
|
||||
@@ -1161,7 +1161,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
// loads them at start, and one missing is a replica that answers for the
|
||||
// base model under the adapter's name.
|
||||
for i, ad := range spec.Profile.Adapters {
|
||||
path, err := a.weights().Ensure(ctx, ad.Path)
|
||||
path, err := a.weights().Ensure(ctx, ad.Path, spec.Profile.AllowPickle)
|
||||
if err != nil {
|
||||
report(supervisor.Failed, fmt.Sprintf("adapter %s: %v", ad.Name, err), nil)
|
||||
return
|
||||
|
||||
@@ -671,7 +671,7 @@ func TestAShardWithMissingWeightsFailsBeforeLaunching(t *testing.T) {
|
||||
// nobody is watching — so it is caught here, where the message can say
|
||||
// what is actually wrong.
|
||||
a := &Agent{cfg: Config{WeightsCache: t.TempDir()}, log: slog.New(slog.DiscardHandler)}
|
||||
_, err := a.weights().Ensure(t.Context(), filepath.Join(t.TempDir(), "not-there.gguf"))
|
||||
_, err := a.weights().Ensure(t.Context(), filepath.Join(t.TempDir(), "not-there.gguf"), false)
|
||||
if err == nil {
|
||||
t.Fatal("a missing weights file was accepted")
|
||||
}
|
||||
|
||||
@@ -47,6 +47,9 @@ type Profile struct {
|
||||
// KVEvents has the engine publish its KV cache events to the agent,
|
||||
// for the router's precise index. vLLM only.
|
||||
KVEvents bool `json:"kv_events,omitempty"`
|
||||
// AllowPickle lets the node fetch weights in a pickle format, which run
|
||||
// code when loaded; the catalog's allow_pickle.
|
||||
AllowPickle bool `json:"allow_pickle,omitempty"`
|
||||
}
|
||||
|
||||
// ShardSpec is one shard of one replica on this node.
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAModelDownloadedByTheEngineIsReadAsSafetensorsOnly(t *testing.T) {
|
||||
for path, want := range map[string]bool{
|
||||
"Qwen/Qwen2.5-7B-Instruct": true,
|
||||
"/var/lib/llmgw/weights/abc/": false, "./local": false,
|
||||
"http://m/q/v/": false, "q.gguf": false,
|
||||
} {
|
||||
got := slices.Contains(safetensorsOnly(Profile{ModelPath: path}), "safetensors")
|
||||
if got != want {
|
||||
t.Errorf("%q: safetensors only = %v, want %v", path, got, want)
|
||||
}
|
||||
}
|
||||
if len(safetensorsOnly(Profile{ModelPath: "Qwen/Qwen2.5-7B-Instruct", AllowPickle: true})) != 0 {
|
||||
t.Error("allow_pickle still forced safetensors")
|
||||
}
|
||||
}
|
||||
@@ -102,6 +102,7 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
|
||||
if p.ServedModelName != "" {
|
||||
args = append(args, "--served-model-name", p.ServedModelName)
|
||||
}
|
||||
args = append(args, safetensorsOnly(p)...)
|
||||
if s.NNodes > 1 {
|
||||
args = append(args,
|
||||
"--nnodes", strconv.Itoa(s.NNodes),
|
||||
|
||||
@@ -2,6 +2,7 @@ package engines
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -185,3 +186,28 @@ func UnsafeArgs(engine string, args []string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// safetensorsOnly has vLLM or SGLang read only safetensors from a model it
|
||||
// downloads itself, by its Hugging Face id: left to choose, either takes
|
||||
// the repository's pickle files (pytorch_model.bin) when it has no
|
||||
// safetensors, and loading those runs whatever they hold. A path was
|
||||
// fetched by the agent, which checked it, or put there by the operator.
|
||||
func safetensorsOnly(p Profile) []string {
|
||||
if p.AllowPickle || !hubID(p.ModelPath) {
|
||||
return nil
|
||||
}
|
||||
return []string{"--load-format", "safetensors"}
|
||||
}
|
||||
|
||||
// hubID reports whether a model path is a Hugging Face id, org/name, rather
|
||||
// than a file or directory.
|
||||
func hubID(s string) bool {
|
||||
if s == "" || strings.HasPrefix(s, "/") || strings.HasPrefix(s, ".") || strings.Contains(s, "://") {
|
||||
return false
|
||||
}
|
||||
if strings.Count(s, "/") != 1 {
|
||||
return false
|
||||
}
|
||||
_, err := os.Stat(s)
|
||||
return err != nil
|
||||
}
|
||||
@@ -119,6 +119,7 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
|
||||
if p.ServedModelName != "" {
|
||||
args = append(args, "--served-model-name", p.ServedModelName)
|
||||
}
|
||||
args = append(args, safetensorsOnly(p)...)
|
||||
|
||||
env := baseEnv(s)
|
||||
|
||||
|
||||
@@ -90,14 +90,20 @@ var ErrNotLocal = errors.New("the weights are not on this node and the source sa
|
||||
// already cached, downloaded, or downloaded by another shard this one waited
|
||||
// for — since that is most of the difference between a replica that starts
|
||||
// in a minute and one that takes an hour.
|
||||
func (f *Fetcher) Ensure(ctx context.Context, source string) (path string, err error) {
|
||||
//
|
||||
// A source in a pickle format, or a set with such a file in it, is refused
|
||||
// before anything is fetched unless allowPickle; see Pickled.
|
||||
func (f *Fetcher) Ensure(ctx context.Context, source string, allowPickle bool) (path string, err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "fetch weights",
|
||||
trace.WithAttributes(attribute.String("llmgw.weights.source", redact(source))))
|
||||
defer func() {
|
||||
span.SetAttributes(attribute.String("llmgw.weights.path", path))
|
||||
obs.End(span, err)
|
||||
}()
|
||||
return f.ensure(ctx, source)
|
||||
if !allowPickle && !IsSet(source) && Pickled(source) {
|
||||
return "", refusePickle(source)
|
||||
}
|
||||
return f.ensure(ctx, source, allowPickle)
|
||||
}
|
||||
|
||||
// tracerName names the spans this package opens.
|
||||
@@ -119,7 +125,7 @@ func redact(source string) string {
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func (f *Fetcher) ensure(ctx context.Context, source string) (string, error) {
|
||||
func (f *Fetcher) ensure(ctx context.Context, source string, allowPickle bool) (string, error) {
|
||||
if source == "" {
|
||||
return "", fmt.Errorf("no source given")
|
||||
}
|
||||
@@ -127,7 +133,7 @@ func (f *Fetcher) ensure(ctx context.Context, source string) (string, error) {
|
||||
// file, and a set has to be handled before any of it.
|
||||
if IsSet(source) {
|
||||
outcome(ctx, "set")
|
||||
return f.ensureSet(ctx, source)
|
||||
return f.ensureSet(ctx, source, allowPickle)
|
||||
}
|
||||
if !isRemote(source) {
|
||||
if !isPath(source) {
|
||||
|
||||
@@ -37,7 +37,7 @@ func TestALocalPathIsUsedNeverCopied(t *testing.T) {
|
||||
}
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
|
||||
got, err := f.Ensure(t.Context(), path)
|
||||
got, err := f.Ensure(t.Context(), path, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -53,7 +53,7 @@ func TestAMissingLocalPathSaysWhatIsWrong(t *testing.T) {
|
||||
// The commonest setup mistake, and "no such file" from an engine twenty
|
||||
// minutes into a load is a poor way to learn it.
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
_, err := f.Ensure(t.Context(), filepath.Join(t.TempDir(), "absent.gguf"))
|
||||
_, err := f.Ensure(t.Context(), filepath.Join(t.TempDir(), "absent.gguf"), false)
|
||||
if !errors.Is(err, ErrNotLocal) {
|
||||
t.Fatalf("err = %v, want ErrNotLocal", err)
|
||||
}
|
||||
@@ -67,7 +67,7 @@ func TestARemoteFileIsFetchedOnceAndReused(t *testing.T) {
|
||||
srv := serveBytes(t, []byte("the weights"), &hits)
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
|
||||
first, err := f.Ensure(t.Context(), srv.URL+"/model.gguf")
|
||||
first, err := f.Ensure(t.Context(), srv.URL+"/model.gguf", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -76,7 +76,7 @@ func TestARemoteFileIsFetchedOnceAndReused(t *testing.T) {
|
||||
t.Fatalf("fetched content = %q, err %v", body, err)
|
||||
}
|
||||
|
||||
second, err := f.Ensure(t.Context(), srv.URL+"/model.gguf")
|
||||
second, err := f.Ensure(t.Context(), srv.URL+"/model.gguf", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -103,7 +103,7 @@ func TestConcurrentRequestsProduceOneDownload(t *testing.T) {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
paths[i], errs[i] = f.Ensure(t.Context(), srv.URL+"/model.gguf")
|
||||
paths[i], errs[i] = f.Ensure(t.Context(), srv.URL+"/model.gguf", false)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
@@ -139,7 +139,7 @@ func TestAPartialDownloadIsNeverVisibleAsTheModel(t *testing.T) {
|
||||
|
||||
cache := t.TempDir()
|
||||
f := &Fetcher{CacheDir: cache}
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf"); err == nil {
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf", false); err == nil {
|
||||
t.Fatal("a truncated download was reported as success")
|
||||
}
|
||||
entries, _ := os.ReadDir(cache)
|
||||
@@ -169,7 +169,7 @@ func TestAnInterruptedDownloadResumes(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := f.Ensure(t.Context(), srv.URL+"/model.gguf")
|
||||
got, err := f.Ensure(t.Context(), srv.URL+"/model.gguf", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -232,7 +232,7 @@ func TestProgressIsReportedWithAPercentage(t *testing.T) {
|
||||
CacheDir: t.TempDir(),
|
||||
OnProgress: func(p Progress) { seen = append(seen, p) },
|
||||
}
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf"); err != nil {
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(seen) == 0 {
|
||||
@@ -254,7 +254,7 @@ func TestPercentIsUnknownRatherThanZeroWithoutATotal(t *testing.T) {
|
||||
|
||||
func TestNoCacheDirectoryIsAClearRefusal(t *testing.T) {
|
||||
f := &Fetcher{}
|
||||
_, err := f.Ensure(t.Context(), "https://example.invalid/model.gguf")
|
||||
_, err := f.Ensure(t.Context(), "https://example.invalid/model.gguf", false)
|
||||
if err == nil || !strings.Contains(err.Error(), "cache directory") {
|
||||
t.Errorf("err = %v, want a complaint about the cache directory", err)
|
||||
}
|
||||
@@ -269,7 +269,7 @@ func TestABareModelIdIsLeftToTheEngine(t *testing.T) {
|
||||
"Qwen/Qwen2.5-7B-Instruct",
|
||||
"meta-llama/Llama-3.3-70B-Instruct",
|
||||
} {
|
||||
got, err := f.Ensure(t.Context(), id)
|
||||
got, err := f.Ensure(t.Context(), id, false)
|
||||
if err != nil {
|
||||
t.Errorf("%s was refused: %v", id, err)
|
||||
}
|
||||
@@ -283,7 +283,7 @@ func TestAnAbsolutePathThatIsMissingIsStillAnError(t *testing.T) {
|
||||
// The other half: an operator writing a path means a file, and a missing
|
||||
// one should be said plainly rather than handed to an engine.
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
if _, err := f.Ensure(t.Context(), "/models/definitely-not-here.gguf"); !errors.Is(err, ErrNotLocal) {
|
||||
if _, err := f.Ensure(t.Context(), "/models/definitely-not-here.gguf", false); !errors.Is(err, ErrNotLocal) {
|
||||
t.Errorf("err = %v, want ErrNotLocal", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package weights
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Weights in Python's pickle format are a program as much as a file: loading
|
||||
// one runs whatever it says, as the engine's user, on the node. That is how
|
||||
// the malicious models found on Hugging Face work, and nothing in the file's
|
||||
// name or size says so. safetensors and GGUF are only tensors and are read,
|
||||
// never run, so those are what a node fetches unless the catalog says
|
||||
// otherwise for a model, with allow_pickle, on purpose.
|
||||
|
||||
// pickleExts are the extensions of PyTorch's pickle formats, and of pickle
|
||||
// itself. A .bin beside safetensors is the same model in pickle form.
|
||||
var pickleExts = []string{".bin", ".pt", ".pth", ".ckpt", ".pkl", ".pickle"}
|
||||
|
||||
// Pickled reports whether a file name or URL is in a pickle format.
|
||||
func Pickled(name string) bool {
|
||||
if i := strings.IndexAny(name, "?#"); i >= 0 && strings.Contains(name, "://") {
|
||||
name = name[:i]
|
||||
}
|
||||
ext := strings.ToLower(path.Ext(name))
|
||||
for _, e := range pickleExts {
|
||||
if ext == e {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ErrPickle is a source refused for being in a pickle format.
|
||||
var ErrPickle = errors.New("weights in a pickle format run code when they are loaded")
|
||||
|
||||
func refusePickle(name string) error {
|
||||
return fmt.Errorf("%w: %s is refused; use the model's safetensors or GGUF, or set allow_pickle: true "+
|
||||
"on the model if it was checked and has no other form", ErrPickle, redact(name))
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package weights
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPickleFormatsAreKnownByTheirNames(t *testing.T) {
|
||||
for name, want := range map[string]bool{
|
||||
"pytorch_model.bin": true, "model.ckpt": true, "x.PT": true, "w.pth": true, "a.pkl": true,
|
||||
"https://h/r/resolve/main/model.ckpt?download=true": true,
|
||||
"model.safetensors": false, "m-Q4_K_M.gguf": false, "Qwen/Qwen2.5-7B-Instruct": false,
|
||||
"config.json": false, "": false,
|
||||
} {
|
||||
if got := Pickled(name); got != want {
|
||||
t.Errorf("Pickled(%q) = %v, want %v", name, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPickleFileIsRefusedBeforeItIsFetched(t *testing.T) {
|
||||
var hits atomic.Int64
|
||||
srv := serveBytes(t, []byte("not really a pickle"), &hits)
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
|
||||
_, err := f.Ensure(t.Context(), srv.URL+"/model.ckpt", false)
|
||||
if !errors.Is(err, ErrPickle) {
|
||||
t.Fatalf("a .ckpt was not refused: %v", err)
|
||||
}
|
||||
if hits.Load() != 0 {
|
||||
t.Errorf("the refused file was fetched %d times", hits.Load())
|
||||
}
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/model.ckpt", true); err != nil {
|
||||
t.Errorf("allowed, the .ckpt was still refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASetWithAPickleFileIsRefusedWhole(t *testing.T) {
|
||||
f, srv, hits := mirrorOf(t, map[string]string{
|
||||
"pytorch_model.bin": "pickle",
|
||||
"config.json": "{}",
|
||||
})
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false); !errors.Is(err, ErrPickle) {
|
||||
t.Fatalf("a set holding pytorch_model.bin was not refused: %v", err)
|
||||
}
|
||||
if *hits != 0 {
|
||||
t.Errorf("%d files of the refused set were fetched", *hits)
|
||||
}
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/", true); err != nil {
|
||||
t.Errorf("allowed, the set was still refused: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -46,7 +46,7 @@ type manifest struct {
|
||||
// The whole set or nothing. A directory with three of four shards in it loads
|
||||
// as a broken model rather than failing to load, so a partial set is left
|
||||
// unfinished under its temporary name and the next attempt starts again.
|
||||
func (f *Fetcher) ensureSet(ctx context.Context, source string) (string, error) {
|
||||
func (f *Fetcher) ensureSet(ctx context.Context, source string, allowPickle bool) (string, error) {
|
||||
dir, err := f.cachePath(strings.TrimSuffix(source, "/"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -66,6 +66,15 @@ func (f *Fetcher) ensureSet(ctx context.Context, source string) (string, error)
|
||||
if len(m.Files) == 0 {
|
||||
return "", fmt.Errorf("%s holds no files", source)
|
||||
}
|
||||
// Refused whole rather than skipped: an engine given the directory
|
||||
// loads the pickle when there is no safetensors beside it.
|
||||
if !allowPickle {
|
||||
for _, file := range m.Files {
|
||||
if Pickled(file.Name) {
|
||||
return "", refusePickle(source + file.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -53,7 +53,7 @@ func TestASetIsFetchedIntoOneDirectory(t *testing.T) {
|
||||
"config.json": "{}",
|
||||
})
|
||||
|
||||
got, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
|
||||
got, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -74,11 +74,11 @@ func TestASetIsFetchedIntoOneDirectory(t *testing.T) {
|
||||
func TestAFinishedSetIsNotFetchedAgain(t *testing.T) {
|
||||
f, srv, hits := mirrorOf(t, map[string]string{"a.gguf": "aaaa", "b.gguf": "bb"})
|
||||
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil {
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after := *hits
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil {
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if *hits != after {
|
||||
@@ -100,7 +100,7 @@ func TestAHalfFetchedSetIsNotTreatedAsFinished(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
|
||||
got, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -117,7 +117,7 @@ func TestAFileAlreadyThereAtTheRightSizeIsNotRefetched(t *testing.T) {
|
||||
_ = os.MkdirAll(dir, 0o755)
|
||||
_ = os.WriteFile(filepath.Join(dir, "a.gguf"), []byte("aaaa"), 0o644)
|
||||
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil {
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Resuming a set means not paying for the shards that already landed.
|
||||
@@ -133,7 +133,7 @@ func TestAFilenameThatEscapesTheDirectoryIsRefused(t *testing.T) {
|
||||
defer srv.Close()
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
|
||||
_, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
|
||||
_, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false)
|
||||
// The mirror is a place an operator copies files into by hand, so what it
|
||||
// reports is not automatically safe to join onto a path.
|
||||
if err == nil || !strings.Contains(err.Error(), "offered a file named") {
|
||||
@@ -148,7 +148,7 @@ func TestADirectoryListingIsNotAManifest(t *testing.T) {
|
||||
defer srv.Close()
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
|
||||
_, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
|
||||
_, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false)
|
||||
// "invalid character '<'" sends somebody looking at the wrong thing.
|
||||
if err == nil || !strings.Contains(err.Error(), "has to be a mirror version") {
|
||||
t.Errorf("the reason should name the cause, got %v", err)
|
||||
@@ -182,12 +182,12 @@ func TestASetFileThatDoesNotMatchTheMirrorsChecksumIsRefused(t *testing.T) {
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err == nil || !strings.Contains(err.Error(), "sha256") {
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false); err == nil || !strings.Contains(err.Error(), "sha256") {
|
||||
t.Fatalf("a damaged shard was accepted: %v", err)
|
||||
}
|
||||
served = good
|
||||
// The size differs, so it is fetched again; this time it matches.
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil {
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false); err != nil {
|
||||
t.Fatalf("the shard as mirrored: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -198,7 +198,7 @@ func TestASetStillDownloadingOnTheMirrorSaysSo(t *testing.T) {
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
_, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
|
||||
_, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false)
|
||||
if err == nil || !strings.Contains(err.Error(), "not complete on the mirror yet") ||
|
||||
!strings.Contains(err.Error(), "model-00002-of-00002") {
|
||||
t.Fatalf("got %v", err)
|
||||
|
||||
@@ -13,7 +13,7 @@ func TestAFetchSaysInItsTraceWhetherItDownloadedAndHowMuch(t *testing.T) {
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
src := strings.Replace(srv.URL, "http://", "http://user:secret@", 1) + "/m/model.gguf?token=hunter2"
|
||||
|
||||
if _, err := f.Ensure(t.Context(), src); err != nil {
|
||||
if _, err := f.Ensure(t.Context(), src, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fetch := rec.One(t, [16]byte{}, "fetch weights")
|
||||
@@ -30,7 +30,7 @@ func TestAFetchSaysInItsTraceWhetherItDownloadedAndHowMuch(t *testing.T) {
|
||||
}
|
||||
|
||||
// The second start on this node finds it here.
|
||||
if _, err := f.Ensure(t.Context(), src); err != nil {
|
||||
if _, err := f.Ensure(t.Context(), src, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again := rec.Named("fetch weights")
|
||||
|
||||
Reference in new issue
Block a user