CI / test (push) Failing after 8m0s
From review. An adapter given by its id is downloaded by vLLM itself, and a TensorRT-LLM model by its id by TensorRT-LLM, past the node's check and not held to safetensors: a model with either is refused when it is added, unless it says allow_pickle, with how to mirror its safetensors instead. vLLM's own model by id stays allowed: the agent gives it --load-format safetensors. A test holds every preset to its own pickle check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
92 lines
2.8 KiB
Go
92 lines
2.8 KiB
Go
package catalog
|
|
|
|
import (
|
|
"fmt"
|
|
"path"
|
|
"slices"
|
|
"strings"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
|
)
|
|
|
|
// PickleSources are the sources of a model's weights and adapters that are
|
|
// in a pickle format, which run code as they are loaded (weights.Pickled).
|
|
func (m Model) PickleSources() []string {
|
|
var out []string
|
|
add := func(s string) {
|
|
if s != "" && weights.Pickled(s) && !slices.Contains(out, s) {
|
|
out = append(out, s)
|
|
}
|
|
}
|
|
add(m.Source)
|
|
for _, p := range m.Profiles {
|
|
add(p.Source)
|
|
}
|
|
for _, a := range m.Adapters {
|
|
add(a.Source)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// CheckPickle refuses a model whose weights are in a pickle format, unless
|
|
// it says allow_pickle. Asked when a model is added or its sources change,
|
|
// not when the catalog is built: a model already running is refused by its
|
|
// node, with the same words, rather than dropped from the catalog.
|
|
// Sources in except were there before and are not asked about again.
|
|
func (m Model) CheckPickle(except ...string) error {
|
|
if m.AllowPickle {
|
|
return nil
|
|
}
|
|
for _, s := range m.PickleSources() {
|
|
if slices.Contains(except, s) {
|
|
continue
|
|
}
|
|
return fmt.Errorf("model %q: %s is in a pickle format, which runs code when it is loaded; "+
|
|
"use the model's safetensors or GGUF, or set allow_pickle: true if it was checked and has no other form",
|
|
m.Name, s)
|
|
}
|
|
// What an engine downloads itself, by a Hugging Face id, no node looks
|
|
// at: vLLM reads a model by id as safetensors only, but not an adapter,
|
|
// and TensorRT-LLM neither.
|
|
for _, s := range m.unlookedSources() {
|
|
if slices.Contains(except, s) {
|
|
continue
|
|
}
|
|
return fmt.Errorf("model %q: %s is a Hugging Face id its engine downloads itself, and may load in a pickle "+
|
|
"format, which runs code; mirror its safetensors and give that, or set allow_pickle: true if it was checked",
|
|
m.Name, s)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// unlookedSources are those an engine fetches by a Hugging Face id, past
|
|
// the node's check: adapters, and a TensorRT-LLM model's.
|
|
func (m Model) unlookedSources() []string {
|
|
var out []string
|
|
for _, a := range m.Adapters {
|
|
if hubID(a.Source) {
|
|
out = append(out, a.Source)
|
|
}
|
|
}
|
|
for _, p := range m.Profiles {
|
|
if p.Engine == "trtllm" {
|
|
if src := m.SourceOf(p); hubID(src) && !slices.Contains(out, src) {
|
|
out = append(out, src)
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// hubID reports whether a source is a Hugging Face id, org/name, rather
|
|
// than a URL, a path or a file.
|
|
func hubID(s string) bool {
|
|
if s == "" || strings.Contains(s, "://") || strings.HasPrefix(s, "/") || strings.HasPrefix(s, ".") ||
|
|
strings.HasPrefix(s, "hf:") || strings.Count(s, "/") != 1 || strings.Contains(s, ":") {
|
|
return false
|
|
}
|
|
// A file named so is a file: org/model.gguf is not a repository.
|
|
ext := strings.ToLower(path.Ext(s))
|
|
return ext != ".gguf" && ext != ".safetensors" && !weights.Pickled(s)
|
|
}
|