Serve vision models with their projector and original weights on vLLM; group the mirror by version
model serve takes --projector, fetched beside the weights and loaded by llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp, which fetches the weights with their config and tokenizer and runs them on vLLM. The console offers both, F switching between GGUF and original weights. The mirror view is one row per version with its files on Enter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
7aefb236d6
commit
a16097cfe8
12 files changed
+509
-81
No files matched your search
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/binary"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -223,3 +224,68 @@ func TestARepositoryThatIsNotOneIsRefused(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAVisionModelIsServedWithItsProjectorBesideIt(t *testing.T) {
|
||||
files := []hf.File{
|
||||
{Path: "gemma-3-4b-it-Q4_K_M.gguf", Bytes: 2 << 30, SHA256: "aa"},
|
||||
{Path: "mmproj-gemma-3-4b-it-f16.gguf", Bytes: 800 << 20, SHA256: "bb"},
|
||||
}
|
||||
s, m, c, _ := setup(t, files, map[string]string{"gemma-3-4b-it-Q4_K_M.gguf": "aa", "mmproj-gemma-3-4b-it-f16.gguf": "bb"})
|
||||
job, err := s.Start(Request{Repo: "google/gemma-3-4b-it-GGUF", Quant: "Q4_K_M",
|
||||
Projector: "mmproj-gemma-3-4b-it-f16.gguf", Replicas: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if j := wait(t, s, job.ID); j.State != Done {
|
||||
t.Fatalf("%s %+v", j.Error, j.Steps)
|
||||
}
|
||||
if len(m.got) != 2 {
|
||||
t.Fatalf("fetched %+v", m.got)
|
||||
}
|
||||
got := c.added[0]
|
||||
if !strings.HasSuffix(got.Source, "/") || !got.Capabilities.Vision || len(got.Origin.Files) != 2 {
|
||||
t.Errorf("%s vision=%v origin=%+v", got.Source, got.Capabilities.Vision, got.Origin)
|
||||
}
|
||||
// A projector the repository does not have is named with those it has.
|
||||
job, _ = s.Start(Request{Repo: "google/gemma-3-4b-it-GGUF", Quant: "Q4_K_M", Projector: "mmproj-x.gguf", Name: "g2"})
|
||||
if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "mmproj-gemma-3-4b-it-f16.gguf") {
|
||||
t.Errorf("%+v", j)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOriginalWeightsAreServedOnVLLMWithTheirConfigAndTokenizer(t *testing.T) {
|
||||
files := []hf.File{
|
||||
{Path: "model-00001-of-00002.safetensors", Bytes: 4 << 30, SHA256: "a1"},
|
||||
{Path: "model-00002-of-00002.safetensors", Bytes: 3 << 30, SHA256: "a2"},
|
||||
{Path: "model.safetensors.index.json", Bytes: 30000},
|
||||
{Path: "config.json", Bytes: 700},
|
||||
{Path: "tokenizer.json", Bytes: 7 << 20},
|
||||
{Path: "tokenizer_config.json", Bytes: 7000},
|
||||
{Path: "README.md", Bytes: 9000},
|
||||
{Path: "original/consolidated.safetensors", Bytes: 7 << 30},
|
||||
}
|
||||
s, m, c, _ := setup(t, files, map[string]string{"model-00001-of-00002.safetensors": "a1", "model-00002-of-00002.safetensors": "a2"})
|
||||
if _, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors"}); err == nil {
|
||||
t.Error("safetensors with no GPU class was taken")
|
||||
}
|
||||
job, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors", GPUClass: "l40s-48g", Replicas: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if j := wait(t, s, job.ID); j.State != Done {
|
||||
t.Fatalf("%s %+v", j.Error, j.Steps)
|
||||
}
|
||||
var got []string
|
||||
for _, r := range m.got {
|
||||
got = append(got, r.File)
|
||||
}
|
||||
if len(got) != 6 || slices.Contains(got, "README.md") || slices.Contains(got, "consolidated.safetensors") {
|
||||
t.Errorf("fetched %v", got)
|
||||
}
|
||||
mod := c.added[0]
|
||||
p := mod.Profiles[0]
|
||||
if !strings.HasSuffix(mod.Source, "/") || p.Engine != "vllm" || p.GPUClass != "l40s-48g" || p.TP != 1 ||
|
||||
!slices.Contains(p.Args, "--max-model-len") {
|
||||
t.Errorf("%s %+v", mod.Source, p)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user