Serve vision models with their projector and original weights on vLLM; group the mirror by version
model serve takes --projector, fetched beside the weights and loaded by llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp, which fetches the weights with their config and tokenizer and runs them on vLLM. The console offers both, F switching between GGUF and original weights. The mirror view is one row per version with its files on Enter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
7aefb236d6
commit
a16097cfe8
12 files changed
+509
-81
No files matched your search
@@ -93,7 +93,7 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
|
||||
bin := l.bin()
|
||||
p := s.Profile
|
||||
|
||||
model, err := ggufIn(p.ModelPath)
|
||||
model, projector, err := ggufIn(p.ModelPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -120,6 +120,11 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
|
||||
if p.ServedModelName != "" {
|
||||
args = append(args, "--alias", p.ServedModelName)
|
||||
}
|
||||
// A vision model's projector, fetched beside it: what turns an image in
|
||||
// a request into something the language model reads.
|
||||
if projector != "" && !hasFlag(p.Args, "--mmproj", "-mm") {
|
||||
args = append(args, "--mmproj", projector)
|
||||
}
|
||||
if !hasFlag(p.Args, "--parallel", "-np") {
|
||||
n := l.cfg.Parallel
|
||||
if n <= 0 {
|
||||
@@ -203,30 +208,46 @@ func hasFlag(args []string, names ...string) bool {
|
||||
//
|
||||
// A path that is already a file is returned untouched, which is every
|
||||
// single-file model and every hand-staged one.
|
||||
func ggufIn(path string) (string, error) {
|
||||
//
|
||||
// A vision projector in the directory — an mmproj file — is not the model:
|
||||
// it is returned second, to be loaded beside it.
|
||||
func ggufIn(path string) (model, projector string, err error) {
|
||||
st, err := os.Stat(path)
|
||||
if err != nil || !st.IsDir() {
|
||||
// Not a directory, or not there at all — the second is the engine's to
|
||||
// report, with the message it would have given anyway.
|
||||
return path, nil //nolint:nilerr // a missing path is the engine's error to give
|
||||
return path, "", nil //nolint:nilerr // a missing path is the engine's error to give
|
||||
}
|
||||
entries, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", "", err
|
||||
}
|
||||
var names []string
|
||||
var names, projectors []string
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), ".gguf") {
|
||||
lower := strings.ToLower(e.Name())
|
||||
if e.IsDir() || !strings.HasSuffix(lower, ".gguf") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(lower, "mmproj") {
|
||||
projectors = append(projectors, e.Name())
|
||||
} else {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return "", fmt.Errorf("%s holds no .gguf file", path)
|
||||
if len(projectors) > 0 {
|
||||
return "", "", fmt.Errorf("%s holds a vision projector and no model", path)
|
||||
}
|
||||
return "", "", fmt.Errorf("%s holds no .gguf file", path)
|
||||
}
|
||||
// Sorted, so a split model's first part comes first: the parts are named
|
||||
// "-00001-of-00004" and lexical order is shard order.
|
||||
sort.Strings(names)
|
||||
return filepath.Join(path, names[0]), nil
|
||||
if len(projectors) > 0 {
|
||||
sort.Strings(projectors)
|
||||
projector = filepath.Join(path, projectors[0])
|
||||
}
|
||||
return filepath.Join(path, names[0]), projector, nil
|
||||
}
|
||||
|
||||
// UseInstalled points the engine at a build the agent installed and checked.
|
||||
|
||||
@@ -204,7 +204,7 @@ func TestASetDirectoryResolvesToItsFirstPart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
got, err := ggufIn(dir)
|
||||
got, _, err := ggufIn(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -221,7 +221,7 @@ func TestAPlainFilePathIsUntouched(t *testing.T) {
|
||||
if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := ggufIn(p); got != p {
|
||||
if got, _, _ := ggufIn(p); got != p {
|
||||
t.Errorf("expected the path unchanged, got %s", got)
|
||||
}
|
||||
}
|
||||
@@ -231,7 +231,7 @@ func TestADirectoryWithNoWeightsSaysSo(t *testing.T) {
|
||||
if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ggufIn(dir); err == nil {
|
||||
if _, _, err := ggufIn(dir); err == nil {
|
||||
t.Error("a directory with no .gguf in it is not a model llama.cpp can load")
|
||||
}
|
||||
}
|
||||
@@ -240,7 +240,7 @@ func TestAMissingPathIsLeftToTheEngine(t *testing.T) {
|
||||
// The engine's own "no such file" names the path and the flag it came
|
||||
// from, which is more use than anything this could say.
|
||||
p := "/nonexistent/model.gguf"
|
||||
got, err := ggufIn(p)
|
||||
got, _, err := ggufIn(p)
|
||||
if err != nil || got != p {
|
||||
t.Errorf("expected it passed through, got %q %v", got, err)
|
||||
}
|
||||
@@ -361,3 +361,28 @@ func TestLlamaCppSpreadsAReplicaOverItsNodesGPUsByLayer(t *testing.T) {
|
||||
t.Error("a one-GPU replica was given a split mode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAVisionProjectorIsLoadedBesideTheModelNotAsIt(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// "mmproj" sorts before the model's name; it must still not be taken for it.
|
||||
for _, n := range []string{"gemma-3-4b-it-Q4_K_M.gguf", "mmproj-gemma-3-4b-it-f16.gguf"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: dir}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
args := strings.Join(cmds[0].Args, " ")
|
||||
if !strings.Contains(args, "--model "+filepath.Join(dir, "gemma-3-4b-it-Q4_K_M.gguf")) ||
|
||||
!strings.Contains(args, "--mmproj "+filepath.Join(dir, "mmproj-gemma-3-4b-it-f16.gguf")) {
|
||||
t.Errorf("%s", args)
|
||||
}
|
||||
// One the profile names itself is left as it is.
|
||||
cmds, _ = (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: dir,
|
||||
Args: []string{"--mmproj", "/elsewhere.gguf"}}})
|
||||
if strings.Count(strings.Join(cmds[0].Args, " "), "--mmproj") != 1 {
|
||||
t.Errorf("two projectors: %v", cmds[0].Args)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user