Weights that run code are refused
Python pickles (.bin, .pt, .pth, .ckpt, .pkl) run whatever they hold when loaded, as the engine's user, on the node; that is how the malicious models found on Hugging Face work. The node refuses such a source, or a mirror set holding one, before fetching anything. A model added with one, or given one, is refused, while one already in the catalog still loads and is refused by its node with the same words. vLLM and SGLang downloading a model by its id get --load-format safetensors. The mirror page marks the file. allow_pickle: true on a model lets it through on purpose, carried to the node in EngineProfile.allow_pickle. BACKLOG M lists the rest: checksums against Hugging Face on every path, its own scan, trusted publishers, engines without the internet, an eval before an untrusted model serves. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6752938e6d
commit
4466d2d15a
31 files changed
+447
-44
No files matched your search
@@ -1124,7 +1124,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
},
|
||||
})
|
||||
})
|
||||
if path, err := a.weights().Ensure(ctx, spec.Profile.ModelPath); err != nil {
|
||||
if path, err := a.weights().Ensure(ctx, spec.Profile.ModelPath, spec.Profile.AllowPickle); err != nil {
|
||||
stop()
|
||||
report(supervisor.Failed, err.Error(), nil)
|
||||
return
|
||||
@@ -1145,7 +1145,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
send(&llmgwv1.ShardStatus{State: agentsvc.StateToWire(supervisor.Starting),
|
||||
Fetch: &llmgwv1.FetchProgress{Source: p.Source, BytesDone: p.BytesDone, BytesTotal: p.BytesTotal}})
|
||||
})
|
||||
path, err := a.weights().Ensure(ctx, m["source"])
|
||||
path, err := a.weights().Ensure(ctx, m["source"], m["allow_pickle"] == "true")
|
||||
stop()
|
||||
if err != nil {
|
||||
report(supervisor.Failed, fmt.Sprintf("model %s: %v", m["name"], err), nil)
|
||||
@@ -1161,7 +1161,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
// loads them at start, and one missing is a replica that answers for the
|
||||
// base model under the adapter's name.
|
||||
for i, ad := range spec.Profile.Adapters {
|
||||
path, err := a.weights().Ensure(ctx, ad.Path)
|
||||
path, err := a.weights().Ensure(ctx, ad.Path, spec.Profile.AllowPickle)
|
||||
if err != nil {
|
||||
report(supervisor.Failed, fmt.Sprintf("adapter %s: %v", ad.Name, err), nil)
|
||||
return
|
||||
|
||||
@@ -671,7 +671,7 @@ func TestAShardWithMissingWeightsFailsBeforeLaunching(t *testing.T) {
|
||||
// nobody is watching — so it is caught here, where the message can say
|
||||
// what is actually wrong.
|
||||
a := &Agent{cfg: Config{WeightsCache: t.TempDir()}, log: slog.New(slog.DiscardHandler)}
|
||||
_, err := a.weights().Ensure(t.Context(), filepath.Join(t.TempDir(), "not-there.gguf"))
|
||||
_, err := a.weights().Ensure(t.Context(), filepath.Join(t.TempDir(), "not-there.gguf"), false)
|
||||
if err == nil {
|
||||
t.Fatal("a missing weights file was accepted")
|
||||
}
|
||||
|
||||
@@ -47,6 +47,9 @@ type Profile struct {
|
||||
// KVEvents has the engine publish its KV cache events to the agent,
|
||||
// for the router's precise index. vLLM only.
|
||||
KVEvents bool `json:"kv_events,omitempty"`
|
||||
// AllowPickle lets the node fetch weights in a pickle format, which run
|
||||
// code when loaded; the catalog's allow_pickle.
|
||||
AllowPickle bool `json:"allow_pickle,omitempty"`
|
||||
}
|
||||
|
||||
// ShardSpec is one shard of one replica on this node.
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAModelDownloadedByTheEngineIsReadAsSafetensorsOnly(t *testing.T) {
|
||||
for path, want := range map[string]bool{
|
||||
"Qwen/Qwen2.5-7B-Instruct": true,
|
||||
"/var/lib/llmgw/weights/abc/": false, "./local": false,
|
||||
"http://m/q/v/": false, "q.gguf": false,
|
||||
} {
|
||||
got := slices.Contains(safetensorsOnly(Profile{ModelPath: path}), "safetensors")
|
||||
if got != want {
|
||||
t.Errorf("%q: safetensors only = %v, want %v", path, got, want)
|
||||
}
|
||||
}
|
||||
if len(safetensorsOnly(Profile{ModelPath: "Qwen/Qwen2.5-7B-Instruct", AllowPickle: true})) != 0 {
|
||||
t.Error("allow_pickle still forced safetensors")
|
||||
}
|
||||
}
|
||||
@@ -102,6 +102,7 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
|
||||
if p.ServedModelName != "" {
|
||||
args = append(args, "--served-model-name", p.ServedModelName)
|
||||
}
|
||||
args = append(args, safetensorsOnly(p)...)
|
||||
if s.NNodes > 1 {
|
||||
args = append(args,
|
||||
"--nnodes", strconv.Itoa(s.NNodes),
|
||||
|
||||
@@ -2,6 +2,7 @@ package engines
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -185,3 +186,28 @@ func UnsafeArgs(engine string, args []string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// safetensorsOnly has vLLM or SGLang read only safetensors from a model it
|
||||
// downloads itself, by its Hugging Face id: left to choose, either takes
|
||||
// the repository's pickle files (pytorch_model.bin) when it has no
|
||||
// safetensors, and loading those runs whatever they hold. A path was
|
||||
// fetched by the agent, which checked it, or put there by the operator.
|
||||
func safetensorsOnly(p Profile) []string {
|
||||
if p.AllowPickle || !hubID(p.ModelPath) {
|
||||
return nil
|
||||
}
|
||||
return []string{"--load-format", "safetensors"}
|
||||
}
|
||||
|
||||
// hubID reports whether a model path is a Hugging Face id, org/name, rather
|
||||
// than a file or directory.
|
||||
func hubID(s string) bool {
|
||||
if s == "" || strings.HasPrefix(s, "/") || strings.HasPrefix(s, ".") || strings.Contains(s, "://") {
|
||||
return false
|
||||
}
|
||||
if strings.Count(s, "/") != 1 {
|
||||
return false
|
||||
}
|
||||
_, err := os.Stat(s)
|
||||
return err != nil
|
||||
}
|
||||
@@ -119,6 +119,7 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
|
||||
if p.ServedModelName != "" {
|
||||
args = append(args, "--served-model-name", p.ServedModelName)
|
||||
}
|
||||
args = append(args, safetensorsOnly(p)...)
|
||||
|
||||
env := baseEnv(s)
|
||||
|
||||
|
||||
@@ -90,14 +90,20 @@ var ErrNotLocal = errors.New("the weights are not on this node and the source sa
|
||||
// already cached, downloaded, or downloaded by another shard this one waited
|
||||
// for — since that is most of the difference between a replica that starts
|
||||
// in a minute and one that takes an hour.
|
||||
func (f *Fetcher) Ensure(ctx context.Context, source string) (path string, err error) {
|
||||
//
|
||||
// A source in a pickle format, or a set with such a file in it, is refused
|
||||
// before anything is fetched unless allowPickle; see Pickled.
|
||||
func (f *Fetcher) Ensure(ctx context.Context, source string, allowPickle bool) (path string, err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "fetch weights",
|
||||
trace.WithAttributes(attribute.String("llmgw.weights.source", redact(source))))
|
||||
defer func() {
|
||||
span.SetAttributes(attribute.String("llmgw.weights.path", path))
|
||||
obs.End(span, err)
|
||||
}()
|
||||
return f.ensure(ctx, source)
|
||||
if !allowPickle && !IsSet(source) && Pickled(source) {
|
||||
return "", refusePickle(source)
|
||||
}
|
||||
return f.ensure(ctx, source, allowPickle)
|
||||
}
|
||||
|
||||
// tracerName names the spans this package opens.
|
||||
@@ -119,7 +125,7 @@ func redact(source string) string {
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func (f *Fetcher) ensure(ctx context.Context, source string) (string, error) {
|
||||
func (f *Fetcher) ensure(ctx context.Context, source string, allowPickle bool) (string, error) {
|
||||
if source == "" {
|
||||
return "", fmt.Errorf("no source given")
|
||||
}
|
||||
@@ -127,7 +133,7 @@ func (f *Fetcher) ensure(ctx context.Context, source string) (string, error) {
|
||||
// file, and a set has to be handled before any of it.
|
||||
if IsSet(source) {
|
||||
outcome(ctx, "set")
|
||||
return f.ensureSet(ctx, source)
|
||||
return f.ensureSet(ctx, source, allowPickle)
|
||||
}
|
||||
if !isRemote(source) {
|
||||
if !isPath(source) {
|
||||
|
||||
@@ -37,7 +37,7 @@ func TestALocalPathIsUsedNeverCopied(t *testing.T) {
|
||||
}
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
|
||||
got, err := f.Ensure(t.Context(), path)
|
||||
got, err := f.Ensure(t.Context(), path, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -53,7 +53,7 @@ func TestAMissingLocalPathSaysWhatIsWrong(t *testing.T) {
|
||||
// The commonest setup mistake, and "no such file" from an engine twenty
|
||||
// minutes into a load is a poor way to learn it.
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
_, err := f.Ensure(t.Context(), filepath.Join(t.TempDir(), "absent.gguf"))
|
||||
_, err := f.Ensure(t.Context(), filepath.Join(t.TempDir(), "absent.gguf"), false)
|
||||
if !errors.Is(err, ErrNotLocal) {
|
||||
t.Fatalf("err = %v, want ErrNotLocal", err)
|
||||
}
|
||||
@@ -67,7 +67,7 @@ func TestARemoteFileIsFetchedOnceAndReused(t *testing.T) {
|
||||
srv := serveBytes(t, []byte("the weights"), &hits)
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
|
||||
first, err := f.Ensure(t.Context(), srv.URL+"/model.gguf")
|
||||
first, err := f.Ensure(t.Context(), srv.URL+"/model.gguf", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -76,7 +76,7 @@ func TestARemoteFileIsFetchedOnceAndReused(t *testing.T) {
|
||||
t.Fatalf("fetched content = %q, err %v", body, err)
|
||||
}
|
||||
|
||||
second, err := f.Ensure(t.Context(), srv.URL+"/model.gguf")
|
||||
second, err := f.Ensure(t.Context(), srv.URL+"/model.gguf", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -103,7 +103,7 @@ func TestConcurrentRequestsProduceOneDownload(t *testing.T) {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
paths[i], errs[i] = f.Ensure(t.Context(), srv.URL+"/model.gguf")
|
||||
paths[i], errs[i] = f.Ensure(t.Context(), srv.URL+"/model.gguf", false)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
@@ -139,7 +139,7 @@ func TestAPartialDownloadIsNeverVisibleAsTheModel(t *testing.T) {
|
||||
|
||||
cache := t.TempDir()
|
||||
f := &Fetcher{CacheDir: cache}
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf"); err == nil {
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf", false); err == nil {
|
||||
t.Fatal("a truncated download was reported as success")
|
||||
}
|
||||
entries, _ := os.ReadDir(cache)
|
||||
@@ -169,7 +169,7 @@ func TestAnInterruptedDownloadResumes(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := f.Ensure(t.Context(), srv.URL+"/model.gguf")
|
||||
got, err := f.Ensure(t.Context(), srv.URL+"/model.gguf", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -232,7 +232,7 @@ func TestProgressIsReportedWithAPercentage(t *testing.T) {
|
||||
CacheDir: t.TempDir(),
|
||||
OnProgress: func(p Progress) { seen = append(seen, p) },
|
||||
}
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf"); err != nil {
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/model.gguf", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(seen) == 0 {
|
||||
@@ -254,7 +254,7 @@ func TestPercentIsUnknownRatherThanZeroWithoutATotal(t *testing.T) {
|
||||
|
||||
func TestNoCacheDirectoryIsAClearRefusal(t *testing.T) {
|
||||
f := &Fetcher{}
|
||||
_, err := f.Ensure(t.Context(), "https://example.invalid/model.gguf")
|
||||
_, err := f.Ensure(t.Context(), "https://example.invalid/model.gguf", false)
|
||||
if err == nil || !strings.Contains(err.Error(), "cache directory") {
|
||||
t.Errorf("err = %v, want a complaint about the cache directory", err)
|
||||
}
|
||||
@@ -269,7 +269,7 @@ func TestABareModelIdIsLeftToTheEngine(t *testing.T) {
|
||||
"Qwen/Qwen2.5-7B-Instruct",
|
||||
"meta-llama/Llama-3.3-70B-Instruct",
|
||||
} {
|
||||
got, err := f.Ensure(t.Context(), id)
|
||||
got, err := f.Ensure(t.Context(), id, false)
|
||||
if err != nil {
|
||||
t.Errorf("%s was refused: %v", id, err)
|
||||
}
|
||||
@@ -283,7 +283,7 @@ func TestAnAbsolutePathThatIsMissingIsStillAnError(t *testing.T) {
|
||||
// The other half: an operator writing a path means a file, and a missing
|
||||
// one should be said plainly rather than handed to an engine.
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
if _, err := f.Ensure(t.Context(), "/models/definitely-not-here.gguf"); !errors.Is(err, ErrNotLocal) {
|
||||
if _, err := f.Ensure(t.Context(), "/models/definitely-not-here.gguf", false); !errors.Is(err, ErrNotLocal) {
|
||||
t.Errorf("err = %v, want ErrNotLocal", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package weights
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Weights in Python's pickle format are a program as much as a file: loading
|
||||
// one runs whatever it says, as the engine's user, on the node. That is how
|
||||
// the malicious models found on Hugging Face work, and nothing in the file's
|
||||
// name or size says so. safetensors and GGUF are only tensors and are read,
|
||||
// never run, so those are what a node fetches unless the catalog says
|
||||
// otherwise for a model, with allow_pickle, on purpose.
|
||||
|
||||
// pickleExts are the extensions of PyTorch's pickle formats, and of pickle
|
||||
// itself. A .bin beside safetensors is the same model in pickle form.
|
||||
var pickleExts = []string{".bin", ".pt", ".pth", ".ckpt", ".pkl", ".pickle"}
|
||||
|
||||
// Pickled reports whether a file name or URL is in a pickle format.
|
||||
func Pickled(name string) bool {
|
||||
if i := strings.IndexAny(name, "?#"); i >= 0 && strings.Contains(name, "://") {
|
||||
name = name[:i]
|
||||
}
|
||||
ext := strings.ToLower(path.Ext(name))
|
||||
for _, e := range pickleExts {
|
||||
if ext == e {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ErrPickle is a source refused for being in a pickle format.
|
||||
var ErrPickle = errors.New("weights in a pickle format run code when they are loaded")
|
||||
|
||||
func refusePickle(name string) error {
|
||||
return fmt.Errorf("%w: %s is refused; use the model's safetensors or GGUF, or set allow_pickle: true "+
|
||||
"on the model if it was checked and has no other form", ErrPickle, redact(name))
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package weights
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPickleFormatsAreKnownByTheirNames(t *testing.T) {
|
||||
for name, want := range map[string]bool{
|
||||
"pytorch_model.bin": true, "model.ckpt": true, "x.PT": true, "w.pth": true, "a.pkl": true,
|
||||
"https://h/r/resolve/main/model.ckpt?download=true": true,
|
||||
"model.safetensors": false, "m-Q4_K_M.gguf": false, "Qwen/Qwen2.5-7B-Instruct": false,
|
||||
"config.json": false, "": false,
|
||||
} {
|
||||
if got := Pickled(name); got != want {
|
||||
t.Errorf("Pickled(%q) = %v, want %v", name, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPickleFileIsRefusedBeforeItIsFetched(t *testing.T) {
|
||||
var hits atomic.Int64
|
||||
srv := serveBytes(t, []byte("not really a pickle"), &hits)
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
|
||||
_, err := f.Ensure(t.Context(), srv.URL+"/model.ckpt", false)
|
||||
if !errors.Is(err, ErrPickle) {
|
||||
t.Fatalf("a .ckpt was not refused: %v", err)
|
||||
}
|
||||
if hits.Load() != 0 {
|
||||
t.Errorf("the refused file was fetched %d times", hits.Load())
|
||||
}
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/model.ckpt", true); err != nil {
|
||||
t.Errorf("allowed, the .ckpt was still refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASetWithAPickleFileIsRefusedWhole(t *testing.T) {
|
||||
f, srv, hits := mirrorOf(t, map[string]string{
|
||||
"pytorch_model.bin": "pickle",
|
||||
"config.json": "{}",
|
||||
})
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false); !errors.Is(err, ErrPickle) {
|
||||
t.Fatalf("a set holding pytorch_model.bin was not refused: %v", err)
|
||||
}
|
||||
if *hits != 0 {
|
||||
t.Errorf("%d files of the refused set were fetched", *hits)
|
||||
}
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/", true); err != nil {
|
||||
t.Errorf("allowed, the set was still refused: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -46,7 +46,7 @@ type manifest struct {
|
||||
// The whole set or nothing. A directory with three of four shards in it loads
|
||||
// as a broken model rather than failing to load, so a partial set is left
|
||||
// unfinished under its temporary name and the next attempt starts again.
|
||||
func (f *Fetcher) ensureSet(ctx context.Context, source string) (string, error) {
|
||||
func (f *Fetcher) ensureSet(ctx context.Context, source string, allowPickle bool) (string, error) {
|
||||
dir, err := f.cachePath(strings.TrimSuffix(source, "/"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -66,6 +66,15 @@ func (f *Fetcher) ensureSet(ctx context.Context, source string) (string, error)
|
||||
if len(m.Files) == 0 {
|
||||
return "", fmt.Errorf("%s holds no files", source)
|
||||
}
|
||||
// Refused whole rather than skipped: an engine given the directory
|
||||
// loads the pickle when there is no safetensors beside it.
|
||||
if !allowPickle {
|
||||
for _, file := range m.Files {
|
||||
if Pickled(file.Name) {
|
||||
return "", refusePickle(source + file.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -53,7 +53,7 @@ func TestASetIsFetchedIntoOneDirectory(t *testing.T) {
|
||||
"config.json": "{}",
|
||||
})
|
||||
|
||||
got, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
|
||||
got, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -74,11 +74,11 @@ func TestASetIsFetchedIntoOneDirectory(t *testing.T) {
|
||||
func TestAFinishedSetIsNotFetchedAgain(t *testing.T) {
|
||||
f, srv, hits := mirrorOf(t, map[string]string{"a.gguf": "aaaa", "b.gguf": "bb"})
|
||||
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil {
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after := *hits
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil {
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if *hits != after {
|
||||
@@ -100,7 +100,7 @@ func TestAHalfFetchedSetIsNotTreatedAsFinished(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
|
||||
got, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -117,7 +117,7 @@ func TestAFileAlreadyThereAtTheRightSizeIsNotRefetched(t *testing.T) {
|
||||
_ = os.MkdirAll(dir, 0o755)
|
||||
_ = os.WriteFile(filepath.Join(dir, "a.gguf"), []byte("aaaa"), 0o644)
|
||||
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil {
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Resuming a set means not paying for the shards that already landed.
|
||||
@@ -133,7 +133,7 @@ func TestAFilenameThatEscapesTheDirectoryIsRefused(t *testing.T) {
|
||||
defer srv.Close()
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
|
||||
_, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
|
||||
_, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false)
|
||||
// The mirror is a place an operator copies files into by hand, so what it
|
||||
// reports is not automatically safe to join onto a path.
|
||||
if err == nil || !strings.Contains(err.Error(), "offered a file named") {
|
||||
@@ -148,7 +148,7 @@ func TestADirectoryListingIsNotAManifest(t *testing.T) {
|
||||
defer srv.Close()
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
|
||||
_, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
|
||||
_, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false)
|
||||
// "invalid character '<'" sends somebody looking at the wrong thing.
|
||||
if err == nil || !strings.Contains(err.Error(), "has to be a mirror version") {
|
||||
t.Errorf("the reason should name the cause, got %v", err)
|
||||
@@ -182,12 +182,12 @@ func TestASetFileThatDoesNotMatchTheMirrorsChecksumIsRefused(t *testing.T) {
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err == nil || !strings.Contains(err.Error(), "sha256") {
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false); err == nil || !strings.Contains(err.Error(), "sha256") {
|
||||
t.Fatalf("a damaged shard was accepted: %v", err)
|
||||
}
|
||||
served = good
|
||||
// The size differs, so it is fetched again; this time it matches.
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil {
|
||||
if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false); err != nil {
|
||||
t.Fatalf("the shard as mirrored: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -198,7 +198,7 @@ func TestASetStillDownloadingOnTheMirrorSaysSo(t *testing.T) {
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
_, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
|
||||
_, err := f.Ensure(t.Context(), srv.URL+"/m/v/", false)
|
||||
if err == nil || !strings.Contains(err.Error(), "not complete on the mirror yet") ||
|
||||
!strings.Contains(err.Error(), "model-00002-of-00002") {
|
||||
t.Fatalf("got %v", err)
|
||||
|
||||
@@ -13,7 +13,7 @@ func TestAFetchSaysInItsTraceWhetherItDownloadedAndHowMuch(t *testing.T) {
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
src := strings.Replace(srv.URL, "http://", "http://user:secret@", 1) + "/m/model.gguf?token=hunter2"
|
||||
|
||||
if _, err := f.Ensure(t.Context(), src); err != nil {
|
||||
if _, err := f.Ensure(t.Context(), src, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fetch := rec.One(t, [16]byte{}, "fetch weights")
|
||||
@@ -30,7 +30,7 @@ func TestAFetchSaysInItsTraceWhetherItDownloadedAndHowMuch(t *testing.T) {
|
||||
}
|
||||
|
||||
// The second start on this node finds it here.
|
||||
if _, err := f.Ensure(t.Context(), src); err != nil {
|
||||
if _, err := f.Ensure(t.Context(), src, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again := rec.Named("fetch weights")
|
||||
|
||||
@@ -201,8 +201,9 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
|
||||
GpusPerNode: int32(p.GPUsPerNode), Nodes: int32(p.Nodes),
|
||||
Args: p.Args, Env: p.Env,
|
||||
ModelPath: p.ModelPath, ServedModelName: p.ServedModelName,
|
||||
Adapters: adaptersToWire(p.Adapters),
|
||||
KvEvents: p.KVEvents,
|
||||
Adapters: adaptersToWire(p.Adapters),
|
||||
KvEvents: p.KVEvents,
|
||||
AllowPickle: p.AllowPickle,
|
||||
},
|
||||
StartupTimeoutMs: cmd.StartupTimeout.Milliseconds(),
|
||||
Pool: cmd.Pool,
|
||||
@@ -238,8 +239,9 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration
|
||||
GPUsPerNode: int(p.GetGpusPerNode()), Nodes: int(p.GetNodes()),
|
||||
Args: p.GetArgs(), Env: p.GetEnv(),
|
||||
ModelPath: p.GetModelPath(), ServedModelName: p.GetServedModelName(),
|
||||
Adapters: adaptersFromWire(p.GetAdapters()),
|
||||
KVEvents: p.GetKvEvents(),
|
||||
Adapters: adaptersFromWire(p.GetAdapters()),
|
||||
KVEvents: p.GetKvEvents(),
|
||||
AllowPickle: p.GetAllowPickle(),
|
||||
},
|
||||
}
|
||||
return spec, time.Duration(m.GetStartupTimeoutMs()) * time.Millisecond
|
||||
|
||||
@@ -323,9 +323,15 @@ type Model struct {
|
||||
// choice by guesswork.
|
||||
Description string `json:"description,omitempty" koanf:"description"`
|
||||
// Source is the weights path or a HuggingFace id.
|
||||
Source string `json:"source,omitempty" koanf:"source"`
|
||||
Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"`
|
||||
Scaling Scaling `json:"scaling,omitempty" koanf:"scaling"`
|
||||
Source string `json:"source,omitempty" koanf:"source"`
|
||||
// AllowPickle lets the nodes fetch this model's weights, or its
|
||||
// adapters', in a pickle format (.bin, .pt, .pth, .ckpt), which run code
|
||||
// as they are loaded. Off, such a source is refused when the model is
|
||||
// added and by the node before it fetches anything. For a model that has
|
||||
// no safetensors or GGUF form and whose files were checked.
|
||||
AllowPickle bool `json:"allow_pickle,omitempty" koanf:"allow_pickle"`
|
||||
Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"`
|
||||
Scaling Scaling `json:"scaling,omitempty" koanf:"scaling"`
|
||||
// Class is how much this model's capacity is worth against another's,
|
||||
// when there is not enough for both: critical, standard or opportunistic.
|
||||
// Empty is standard, or opportunistic for a model loaded on demand. See
|
||||
@@ -1336,10 +1342,16 @@ func (m Model) EngineProfile(name string) (engines.Profile, bool) {
|
||||
Name string `json:"name"`
|
||||
Source string `json:"source"`
|
||||
HFRepo string `json:"hf_repo,omitempty"`
|
||||
// A string: the node reads the members as strings.
|
||||
AllowPickle string `json:"allow_pickle,omitempty"`
|
||||
}
|
||||
var ms []member
|
||||
for _, sm := range m.Shared.Members {
|
||||
ms = append(ms, member{Name: sm.Name, Source: sm.Source, HFRepo: sm.HFRepo})
|
||||
mb := member{Name: sm.Name, Source: sm.Source, HFRepo: sm.HFRepo}
|
||||
if sm.AllowPickle {
|
||||
mb.AllowPickle = "true"
|
||||
}
|
||||
ms = append(ms, mb)
|
||||
}
|
||||
b, _ := json.Marshal(ms)
|
||||
env = maps.Clone(env)
|
||||
@@ -1369,6 +1381,7 @@ func (m Model) EngineProfile(name string) (engines.Profile, bool) {
|
||||
ServedModelName: m.Name,
|
||||
Adapters: m.engineAdapters(),
|
||||
KVEvents: p.KVEvents,
|
||||
AllowPickle: m.AllowPickle,
|
||||
}.WithRuntimeLoRA(m.MaxLoadedAdapters), true
|
||||
}
|
||||
return engines.Profile{}, false
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
||||
)
|
||||
|
||||
// PickleSources are the sources of a model's weights and adapters that are
|
||||
// in a pickle format, which run code as they are loaded (weights.Pickled).
|
||||
func (m Model) PickleSources() []string {
|
||||
var out []string
|
||||
add := func(s string) {
|
||||
if s != "" && weights.Pickled(s) && !slices.Contains(out, s) {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
add(m.Source)
|
||||
for _, p := range m.Profiles {
|
||||
add(p.Source)
|
||||
}
|
||||
for _, a := range m.Adapters {
|
||||
add(a.Source)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CheckPickle refuses a model whose weights are in a pickle format, unless
|
||||
// it says allow_pickle. Asked when a model is added or its sources change,
|
||||
// not when the catalog is built: a model already running is refused by its
|
||||
// node, with the same words, rather than dropped from the catalog.
|
||||
// Sources in except were there before and are not asked about again.
|
||||
func (m Model) CheckPickle(except ...string) error {
|
||||
if m.AllowPickle {
|
||||
return nil
|
||||
}
|
||||
for _, s := range m.PickleSources() {
|
||||
if slices.Contains(except, s) {
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("model %q: %s is in a pickle format, which runs code when it is loaded; "+
|
||||
"use the model's safetensors or GGUF, or set allow_pickle: true if it was checked and has no other form",
|
||||
m.Name, s)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAModelInAPickleFormatIsRefusedUnlessItSaysSo(t *testing.T) {
|
||||
m := Model{Name: "triposr", Source: "hf:stabilityai/TripoSR/model.ckpt",
|
||||
Adapters: []Adapter{{Name: "a", Source: "http://m/a/adapter_model.bin"}}}
|
||||
err := m.CheckPickle()
|
||||
if err == nil || !strings.Contains(err.Error(), "model.ckpt") || !strings.Contains(err.Error(), "allow_pickle") {
|
||||
t.Fatalf("not refused, or not said why: %v", err)
|
||||
}
|
||||
if got := m.PickleSources(); len(got) != 2 {
|
||||
t.Errorf("pickle sources %v, want the model's and the adapter's", got)
|
||||
}
|
||||
// A source that was there before is not asked about again: changing
|
||||
// the scaling of a model that runs does not need allow_pickle.
|
||||
if err := m.CheckPickle(m.PickleSources()...); err != nil {
|
||||
t.Errorf("sources already there were refused: %v", err)
|
||||
}
|
||||
m.AllowPickle = true
|
||||
if err := m.CheckPickle(); err != nil {
|
||||
t.Errorf("allow_pickle was not honoured: %v", err)
|
||||
}
|
||||
if err := (Model{Name: "q", Source: "http://m/q/v/q-Q4_K_M.gguf"}).CheckPickle(); err != nil {
|
||||
t.Errorf("a GGUF was refused: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,8 @@ type SharedMember struct {
|
||||
Source string `json:"source"`
|
||||
// HFRepo is the repository its node loads the weights by name from.
|
||||
HFRepo string `json:"hf_repo,omitempty"`
|
||||
// AllowPickle is the member model's allow_pickle.
|
||||
AllowPickle bool `json:"allow_pickle,omitempty"`
|
||||
// Workflow and Checkpoint are what a request for it is sent.
|
||||
Workflow string `json:"workflow"`
|
||||
Checkpoint string `json:"checkpoint"`
|
||||
@@ -80,7 +82,7 @@ func envModel(env Environment, members []Model) Model {
|
||||
hf = m.ComfyUI.HuggingFaceRepo
|
||||
}
|
||||
shared.Members = append(shared.Members, SharedMember{Name: m.Name, Aliases: m.Aliases,
|
||||
Source: m.Source, HFRepo: hf, Workflow: workflow, Checkpoint: ckpt})
|
||||
Source: m.Source, HFRepo: hf, AllowPickle: m.AllowPickle, Workflow: workflow, Checkpoint: ckpt})
|
||||
idle = max(idle, m.Scaling.IdleTimeout)
|
||||
minN, maxN = max(minN, m.Scaling.Min), max(maxN, m.Scaling.Max)
|
||||
if ClassRank(m.EffectiveClass()) > ClassRank(class) || class == "" {
|
||||
|
||||
@@ -154,6 +154,9 @@ func (s *Service) add(ctx context.Context, m catalog.Model, by string, onlyNew b
|
||||
next = append(next, e)
|
||||
}
|
||||
next = append(next, m)
|
||||
if err := m.CheckPickle(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
built, err := catalog.New(next)
|
||||
if err != nil {
|
||||
@@ -406,9 +409,13 @@ func (s *Service) update(ctx context.Context, name, by, what string, change func
|
||||
found := false
|
||||
for _, e := range stored {
|
||||
if e.Name == name {
|
||||
before := e.PickleSources()
|
||||
if err := change(&e); err != nil {
|
||||
return catalog.Model{}, err
|
||||
}
|
||||
if err := e.CheckPickle(before...); err != nil {
|
||||
return catalog.Model{}, err
|
||||
}
|
||||
found = true
|
||||
}
|
||||
next = append(next, e)
|
||||
|
||||
@@ -240,6 +240,7 @@ var Reference = []Block{
|
||||
{Key: "aliases", Type: "[]string", Doc: "", Nested: ""},
|
||||
{Key: "description", Type: "string", Doc: "Description is what this model is for, in a line.\n\nWritten by whoever added it, because nobody else can say it. A registry\noffers a pipeline tag and a wall of README, and neither answers the\nquestion a person picking from a list is asking: which of these nine\nnames should I send this request to.\n\nIt reaches clients through /v1/models, which is the only view most of\nthem ever see. A gateway serving nine models by name alone makes that\nchoice by guesswork.", Nested: ""},
|
||||
{Key: "source", Type: "string", Doc: "Source is the weights path or a HuggingFace id.", Nested: ""},
|
||||
{Key: "allow_pickle", Type: "bool", Doc: "AllowPickle lets the nodes fetch this model's weights, or its\nadapters', in a pickle format (.bin, .pt, .pth, .ckpt), which run code\nas they are loaded. Off, such a source is refused when the model is\nadded and by the node before it fetches anything. For a model that has\nno safetensors or GGUF form and whose files were checked.", Nested: ""},
|
||||
{Key: "profiles", Type: "[]Profile", Doc: "", Nested: ""},
|
||||
{Key: "scaling", Type: "Scaling", Doc: "", Nested: "catalog.Scaling"},
|
||||
{Key: "class", Type: "string", Doc: "Class is how much this model's capacity is worth against another's,\nwhen there is not enough for both: critical, standard or opportunistic.\nEmpty is standard, or opportunistic for a model loaded on demand. See\nEffectiveClass.", Nested: ""},
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
@@ -876,6 +877,11 @@ func defaultModelName(repoID string) string { return hf.ModelName(repoID) }
|
||||
// verdictFor judges a registry file set by the repository's architecture. A
|
||||
// vision projector is its own architecture whatever the repository says.
|
||||
func (p *AccountPages) verdictFor(l *Lang, arch, name string) gguf.Verdict {
|
||||
// Not refused here — the mirror keeps what it is given — but a node
|
||||
// refuses to fetch it for a model without allow_pickle.
|
||||
if weights.Pickled(name) {
|
||||
return gguf.Verdict{Checked: true, Reason: l.T("card.pickle")}
|
||||
}
|
||||
if strings.Contains(strings.ToLower(name), "mmproj") {
|
||||
return gguf.LlamaCpp(p.llamaVersion, "clip")
|
||||
}
|
||||
|
||||
@@ -137,6 +137,7 @@
|
||||
"card.model": "model",
|
||||
"card.cannot_serve": "cannot be served",
|
||||
"card.imatrix": "it is an importance matrix for quantising, not a model",
|
||||
"card.pickle": "it is in a pickle format, which runs code when it is loaded; nodes refuse it unless the model says allow_pickle — take the safetensors or GGUF",
|
||||
"card.fetch": "Fetch",
|
||||
"card.weights_note": "Only the weights are listed; a repository is mostly a README and a tokenizer. A safetensors set is fetched with the files vLLM reads beside it — config.json, the tokenizer, the shard index — and never with code. Shards of one file are one row and one button — a fifteen-shard model is one decision, not fifteen. Two separate files are two rows because they are usually alternatives: one quantisation or the other, not both.",
|
||||
"card.pinned_note": "The source URLs are pinned to <code>%s</code> rather than to a branch, so what arrives is what is listed here even if the branch moves in between — and a single file's checksum above is what the registry publishes, to compare with what the mirror computes once it has it.",
|
||||
|
||||
@@ -132,6 +132,7 @@
|
||||
"card.model": "model",
|
||||
"card.cannot_serve": "sunulamaz",
|
||||
"card.imatrix": "bu bir model değil, nicemleme için bir önem matrisi",
|
||||
"card.pickle": "pickle biçiminde; yüklenirken kod çalıştırır. Model allow_pickle demedikçe düğümler onu almaz — safetensors ya da GGUF olanı seçin",
|
||||
"card.fetch": "İndir",
|
||||
"card.weights_note": "Yalnızca ağırlıklar listelenir; bir deponun çoğu bir README ve bir tokenizer'dır. Bir safetensors kümesi, vLLM'in yanında okuduğu dosyalarla — config.json, tokenizer, parça dizini — birlikte indirilir, asla kodla birlikte değil. Bir dosyanın parçaları tek satır ve tek düğmedir — on beş parçalı bir model on beş değil, tek bir karardır. Ayrı iki dosya iki satırdır, çünkü genellikle birbirinin seçeneğidir: ya bir nicemleme ya öteki, ikisi birden değil.",
|
||||
"card.pinned_note": "Kaynak URL'ler bir dala değil <code>%s</code> sürümüne sabitlenmiştir; böylece dal arada değişse bile gelen, burada listelenendir — yukarıdaki tek dosya sağlama toplamı da model deposunun yayımladığıdır ve ayna dosyayı aldığında hesapladığıyla karşılaştırılabilir.",
|
||||
|
||||
Reference in new issue
Block a user