Serve vision models with their projector and original weights on vLLM; group the mirror by version

model serve takes --projector, fetched beside the weights and loaded by
llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp,
which fetches the weights with their config and tokenizer and runs them on
vLLM. The console offers both, F switching between GGUF and original
weights. The mirror view is one row per version with its files on Enter.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 00:42:59 +02:00
1 parent 7aefb236d6
commit a16097cfe8
12 files changed
+509 -81

No files matched your search

+29 -8
View File
@@ -93,7 +93,7 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
bin := l.bin()
p := s.Profile
model, err := ggufIn(p.ModelPath)
model, projector, err := ggufIn(p.ModelPath)
if err != nil {
return nil, err
}
@@ -120,6 +120,11 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
if p.ServedModelName != "" {
args = append(args, "--alias", p.ServedModelName)
}
// A vision model's projector, fetched beside it: what turns an image in
// a request into something the language model reads.
if projector != "" && !hasFlag(p.Args, "--mmproj", "-mm") {
args = append(args, "--mmproj", projector)
}
if !hasFlag(p.Args, "--parallel", "-np") {
n := l.cfg.Parallel
if n <= 0 {
@@ -203,30 +208,46 @@ func hasFlag(args []string, names ...string) bool {
//
// A path that is already a file is returned untouched, which is every
// single-file model and every hand-staged one.
func ggufIn(path string) (string, error) {
//
// A vision projector in the directory — an mmproj file — is not the model:
// it is returned second, to be loaded beside it.
func ggufIn(path string) (model, projector string, err error) {
st, err := os.Stat(path)
if err != nil || !st.IsDir() {
// Not a directory, or not there at all — the second is the engine's to
// report, with the message it would have given anyway.
return path, nil //nolint:nilerr // a missing path is the engine's error to give
return path, "", nil //nolint:nilerr // a missing path is the engine's error to give
}
entries, err := os.ReadDir(path)
if err != nil {
return "", err
return "", "", err
}
var names []string
var names, projectors []string
for _, e := range entries {
if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), ".gguf") {
lower := strings.ToLower(e.Name())
if e.IsDir() || !strings.HasSuffix(lower, ".gguf") {
continue
}
if strings.Contains(lower, "mmproj") {
projectors = append(projectors, e.Name())
} else {
names = append(names, e.Name())
}
}
if len(names) == 0 {
return "", fmt.Errorf("%s holds no .gguf file", path)
if len(projectors) > 0 {
return "", "", fmt.Errorf("%s holds a vision projector and no model", path)
}
return "", "", fmt.Errorf("%s holds no .gguf file", path)
}
// Sorted, so a split model's first part comes first: the parts are named
// "-00001-of-00004" and lexical order is shard order.
sort.Strings(names)
return filepath.Join(path, names[0]), nil
if len(projectors) > 0 {
sort.Strings(projectors)
projector = filepath.Join(path, projectors[0])
}
return filepath.Join(path, names[0]), projector, nil
}
// UseInstalled points the engine at a build the agent installed and checked.
+29 -4
View File
@@ -204,7 +204,7 @@ func TestASetDirectoryResolvesToItsFirstPart(t *testing.T) {
}
}
got, err := ggufIn(dir)
got, _, err := ggufIn(dir)
if err != nil {
t.Fatal(err)
}
@@ -221,7 +221,7 @@ func TestAPlainFilePathIsUntouched(t *testing.T) {
if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if got, _ := ggufIn(p); got != p {
if got, _, _ := ggufIn(p); got != p {
t.Errorf("expected the path unchanged, got %s", got)
}
}
@@ -231,7 +231,7 @@ func TestADirectoryWithNoWeightsSaysSo(t *testing.T) {
if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{}"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := ggufIn(dir); err == nil {
if _, _, err := ggufIn(dir); err == nil {
t.Error("a directory with no .gguf in it is not a model llama.cpp can load")
}
}
@@ -240,7 +240,7 @@ func TestAMissingPathIsLeftToTheEngine(t *testing.T) {
// The engine's own "no such file" names the path and the flag it came
// from, which is more use than anything this could say.
p := "/nonexistent/model.gguf"
got, err := ggufIn(p)
got, _, err := ggufIn(p)
if err != nil || got != p {
t.Errorf("expected it passed through, got %q %v", got, err)
}
@@ -361,3 +361,28 @@ func TestLlamaCppSpreadsAReplicaOverItsNodesGPUsByLayer(t *testing.T) {
t.Error("a one-GPU replica was given a split mode")
}
}
func TestAVisionProjectorIsLoadedBesideTheModelNotAsIt(t *testing.T) {
dir := t.TempDir()
// "mmproj" sorts before the model's name; it must still not be taken for it.
for _, n := range []string{"gemma-3-4b-it-Q4_K_M.gguf", "mmproj-gemma-3-4b-it-f16.gguf"} {
if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
}
cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: dir}})
if err != nil {
t.Fatal(err)
}
args := strings.Join(cmds[0].Args, " ")
if !strings.Contains(args, "--model "+filepath.Join(dir, "gemma-3-4b-it-Q4_K_M.gguf")) ||
!strings.Contains(args, "--mmproj "+filepath.Join(dir, "mmproj-gemma-3-4b-it-f16.gguf")) {
t.Errorf("%s", args)
}
// One the profile names itself is left as it is.
cmds, _ = (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, Profile: Profile{ModelPath: dir,
Args: []string{"--mmproj", "/elsewhere.gguf"}}})
if strings.Count(strings.Join(cmds[0].Args, " "), "--mmproj") != 1 {
t.Errorf("two projectors: %v", cmds[0].Args)
}
}