Serve vision models with their projector and original weights on vLLM; group the mirror by version

model serve takes --projector, fetched beside the weights and loaded by
llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp,
which fetches the weights with their config and tokenizer and runs them on
vLLM. The console offers both, F switching between GGUF and original
weights. The mirror view is one row per version with its files on Enter.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 00:42:59 +02:00
1 parent 7aefb236d6
commit a16097cfe8
12 files changed
+509 -81

No files matched your search

@@ -6,6 +6,7 @@ import (
"encoding/binary"
"net/http"
"net/http/httptest"
"slices"
"strings"
"sync"
"testing"
@@ -223,3 +224,68 @@ func TestARepositoryThatIsNotOneIsRefused(t *testing.T) {
}
}
}
func TestAVisionModelIsServedWithItsProjectorBesideIt(t *testing.T) {
files := []hf.File{
{Path: "gemma-3-4b-it-Q4_K_M.gguf", Bytes: 2 << 30, SHA256: "aa"},
{Path: "mmproj-gemma-3-4b-it-f16.gguf", Bytes: 800 << 20, SHA256: "bb"},
}
s, m, c, _ := setup(t, files, map[string]string{"gemma-3-4b-it-Q4_K_M.gguf": "aa", "mmproj-gemma-3-4b-it-f16.gguf": "bb"})
job, err := s.Start(Request{Repo: "google/gemma-3-4b-it-GGUF", Quant: "Q4_K_M",
Projector: "mmproj-gemma-3-4b-it-f16.gguf", Replicas: 1})
if err != nil {
t.Fatal(err)
}
if j := wait(t, s, job.ID); j.State != Done {
t.Fatalf("%s %+v", j.Error, j.Steps)
}
if len(m.got) != 2 {
t.Fatalf("fetched %+v", m.got)
}
got := c.added[0]
if !strings.HasSuffix(got.Source, "/") || !got.Capabilities.Vision || len(got.Origin.Files) != 2 {
t.Errorf("%s vision=%v origin=%+v", got.Source, got.Capabilities.Vision, got.Origin)
}
// A projector the repository does not have is named with those it has.
job, _ = s.Start(Request{Repo: "google/gemma-3-4b-it-GGUF", Quant: "Q4_K_M", Projector: "mmproj-x.gguf", Name: "g2"})
if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "mmproj-gemma-3-4b-it-f16.gguf") {
t.Errorf("%+v", j)
}
}
func TestOriginalWeightsAreServedOnVLLMWithTheirConfigAndTokenizer(t *testing.T) {
files := []hf.File{
{Path: "model-00001-of-00002.safetensors", Bytes: 4 << 30, SHA256: "a1"},
{Path: "model-00002-of-00002.safetensors", Bytes: 3 << 30, SHA256: "a2"},
{Path: "model.safetensors.index.json", Bytes: 30000},
{Path: "config.json", Bytes: 700},
{Path: "tokenizer.json", Bytes: 7 << 20},
{Path: "tokenizer_config.json", Bytes: 7000},
{Path: "README.md", Bytes: 9000},
{Path: "original/consolidated.safetensors", Bytes: 7 << 30},
}
s, m, c, _ := setup(t, files, map[string]string{"model-00001-of-00002.safetensors": "a1", "model-00002-of-00002.safetensors": "a2"})
if _, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors"}); err == nil {
t.Error("safetensors with no GPU class was taken")
}
job, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors", GPUClass: "l40s-48g", Replicas: 1})
if err != nil {
t.Fatal(err)
}
if j := wait(t, s, job.ID); j.State != Done {
t.Fatalf("%s %+v", j.Error, j.Steps)
}
var got []string
for _, r := range m.got {
got = append(got, r.File)
}
if len(got) != 6 || slices.Contains(got, "README.md") || slices.Contains(got, "consolidated.safetensors") {
t.Errorf("fetched %v", got)
}
mod := c.added[0]
p := mod.Profiles[0]
if !strings.HasSuffix(mod.Source, "/") || p.Engine != "vllm" || p.GPUClass != "l40s-48g" || p.TP != 1 ||
!slices.Contains(p.Args, "--max-model-len") {
t.Errorf("%s %+v", mod.Source, p)
}
}