package weights import ( "errors" "fmt" "path" "strings" ) // Weights in Python's pickle format are a program as much as a file: loading // one runs whatever it says, as the engine's user, on the node. That is how // the malicious models found on Hugging Face work, and nothing in the file's // name or size says so. safetensors and GGUF are only tensors and are read, // never run, so those are what a node fetches unless the catalog says // otherwise for a model, with allow_pickle, on purpose. // pickleExts are the extensions of PyTorch's pickle formats, and of pickle // itself. A .bin beside safetensors is the same model in pickle form. var pickleExts = []string{".bin", ".pt", ".pt2", ".pth", ".ckpt", ".pkl", ".pickle"} // Pickled reports whether a file name or URL is in a pickle format. func Pickled(name string) bool { // A query or a fragment is not the file's name, on a URL or an hf: // source alike: model.bin?download=true is still model.bin. if i := strings.IndexAny(name, "?#"); i >= 0 { name = name[:i] } ext := strings.ToLower(path.Ext(name)) for _, e := range pickleExts { if ext == e { return true } } return false } // ErrPickle is a source refused for being in a pickle format. var ErrPickle = errors.New("weights in a pickle format run code when they are loaded") func refusePickle(name string) error { return fmt.Errorf("%w: %s is refused; use the model's safetensors or GGUF, or set allow_pickle: true "+ "on the model if it was checked and has no other form", ErrPickle, redact(name)) }