Serve vision models with their projector and original weights on vLLM; group the mirror by version
model serve takes --projector, fetched beside the weights and loaded by llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp, which fetches the weights with their config and tokenizer and runs them on vLLM. The console offers both, F switching between GGUF and original weights. The mirror view is one row per version with its files on Enter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
7aefb236d6
commit
a16097cfe8
12 files changed
+509
-81
No files matched your search
@@ -87,6 +87,15 @@ type Request struct {
|
||||
Quant string `json:"quant,omitempty"`
|
||||
// File picks it by name instead.
|
||||
File string `json:"file,omitempty"`
|
||||
// Projector is a vision model's mmproj file in the repository, fetched
|
||||
// beside the weights and loaded with them; empty serves text alone.
|
||||
Projector string `json:"projector,omitempty"`
|
||||
// Format is "gguf", run by llama.cpp, or "safetensors": the original
|
||||
// weights with their config and tokenizer, run by vLLM on GPUs of
|
||||
// GPUClass, across TP of them.
|
||||
Format string `json:"format,omitempty"`
|
||||
GPUClass string `json:"gpu_class,omitempty"`
|
||||
TP int `json:"tp,omitempty"`
|
||||
// Name is the catalog name; the repository suggests one otherwise.
|
||||
Name string `json:"name,omitempty"`
|
||||
Aliases []string `json:"aliases,omitempty"`
|
||||
@@ -179,6 +188,20 @@ func (s *Server) Start(req Request) (Job, error) {
|
||||
if req.Replicas < 0 {
|
||||
return Job{}, errors.New("replicas cannot be negative")
|
||||
}
|
||||
switch req.Format {
|
||||
case "", "gguf":
|
||||
req.Format = "gguf"
|
||||
case "safetensors":
|
||||
if req.GPUClass == "" {
|
||||
return Job{}, errors.New("safetensors weights run on vLLM, on GPUs: name the GPU class, as `llmgw ctl status` lists it")
|
||||
}
|
||||
if req.Projector != "" {
|
||||
return Job{}, errors.New("a projector is for a GGUF model; vLLM reads a vision model's own files")
|
||||
}
|
||||
req.TP = max(req.TP, 1)
|
||||
default:
|
||||
return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format)
|
||||
}
|
||||
if req.Quant == "" && req.File == "" {
|
||||
req.Quant = "Q4_K_M"
|
||||
}
|
||||
@@ -251,11 +274,28 @@ func (s *Server) run(j *Job) {
|
||||
s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", req.Repo))
|
||||
return
|
||||
}
|
||||
set, err := pick(repo, req)
|
||||
pickSet := pick
|
||||
if req.Format == "safetensors" {
|
||||
pickSet = pickSafetensors
|
||||
}
|
||||
set, err := pickSet(repo, req)
|
||||
if err != nil {
|
||||
s.fail(j, "registry", err)
|
||||
return
|
||||
}
|
||||
if req.Projector != "" {
|
||||
i := slices.IndexFunc(repo.Files, func(f hf.File) bool {
|
||||
return (f.Path == req.Projector || mirror.LastPathElement(f.Path) == req.Projector) &&
|
||||
strings.Contains(strings.ToLower(f.Path), "mmproj")
|
||||
})
|
||||
if i < 0 {
|
||||
s.fail(j, "registry", fmt.Errorf("%s has no vision projector %q; it has: %s", repo.ID, req.Projector,
|
||||
strings.Join(Projectors(repo), ", ")))
|
||||
return
|
||||
}
|
||||
set.Files = append(set.Files, repo.Files[i])
|
||||
set.Bytes += repo.Files[i].Bytes
|
||||
}
|
||||
name := req.Name
|
||||
if name == "" {
|
||||
name = hf.ModelName(repo.ID)
|
||||
@@ -274,10 +314,20 @@ func (s *Server) run(j *Job) {
|
||||
}
|
||||
|
||||
// 3. Whether the nodes' llama.cpp loads it, and how much memory it needs.
|
||||
if req.Format == "safetensors" {
|
||||
// vLLM reads the model's shape from its config.json, which came
|
||||
// with the weights; the GPUs it needs are the class and width asked.
|
||||
s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM on %d × %s", gb(set.Bytes), req.TP, req.GPUClass))
|
||||
s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{
|
||||
Name: "gpu", Engine: "vllm", GPUClass: req.GPUClass, TP: req.TP, Weight: 1, Priority: 100,
|
||||
Args: []string{"--max-model-len", strconv.Itoa(req.ContextLength)},
|
||||
})
|
||||
return
|
||||
}
|
||||
s.step(j, "check", Running, "reading its header from the mirror")
|
||||
var header gguf.Header
|
||||
if !strings.HasSuffix(source, "/") {
|
||||
v := gguf.Check(ctx, source, s.deps.LlamaVersion)
|
||||
if u := headerURL(source, set); u != "" {
|
||||
v := gguf.Check(ctx, u, s.deps.LlamaVersion)
|
||||
if !v.Servable {
|
||||
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run this: %s", s.deps.LlamaVersion, v.Reason))
|
||||
return
|
||||
@@ -302,16 +352,24 @@ func (s *Server) run(j *Job) {
|
||||
}
|
||||
}
|
||||
|
||||
s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{
|
||||
Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100,
|
||||
Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores,
|
||||
})
|
||||
}
|
||||
|
||||
// addAndStart puts the model in the catalog with its profile, waits for a
|
||||
// replica and asks it to answer.
|
||||
func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, prof catalog.Profile) {
|
||||
req := j.Request
|
||||
// 4. The catalog.
|
||||
m := catalog.Model{
|
||||
Name: name, Aliases: req.Aliases, Description: req.Description, Source: source,
|
||||
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024,
|
||||
Profiles: []catalog.Profile{{
|
||||
Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100,
|
||||
Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores,
|
||||
}},
|
||||
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
|
||||
Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)},
|
||||
Profiles: []catalog.Profile{prof},
|
||||
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
|
||||
Capabilities: catalog.Capabilities{Vision: req.Projector != ""},
|
||||
Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)},
|
||||
}
|
||||
s.step(j, "catalog", Running, "adding "+name)
|
||||
if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil {
|
||||
@@ -593,3 +651,72 @@ func orUnknown(arch string) string {
|
||||
}
|
||||
return arch
|
||||
}
|
||||
|
||||
// Projectors are a repository's vision projectors: the mmproj files a vision
|
||||
// model loads beside its weights.
|
||||
func Projectors(repo hf.Repo) []string {
|
||||
var out []string
|
||||
for _, f := range repo.Files {
|
||||
if strings.HasSuffix(strings.ToLower(f.Path), ".gguf") && strings.Contains(strings.ToLower(f.Path), "mmproj") {
|
||||
out = append(out, f.Path)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// headerURL is the file whose header says what the model is: the source
|
||||
// itself, or in a directory the first part of the weights, not a projector.
|
||||
func headerURL(source string, set hf.FileSet) string {
|
||||
if !strings.HasSuffix(source, "/") {
|
||||
return source
|
||||
}
|
||||
var names []string
|
||||
for _, f := range set.Files {
|
||||
name := mirror.LastPathElement(f.Path)
|
||||
lower := strings.ToLower(name)
|
||||
if strings.HasSuffix(lower, ".gguf") && !strings.Contains(lower, "mmproj") {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return ""
|
||||
}
|
||||
slices.Sort(names)
|
||||
return source + names[0]
|
||||
}
|
||||
|
||||
// SafetensorsSet is what serving a repository's original weights fetches.
|
||||
func SafetensorsSet(repo hf.Repo) (hf.FileSet, error) { return pickSafetensors(repo, Request{}) }
|
||||
|
||||
// pickSafetensors chooses a repository's original weights and the files an
|
||||
// engine reads beside them: config.json, the tokenizer, the shard index.
|
||||
func pickSafetensors(repo hf.Repo, _ Request) (hf.FileSet, error) {
|
||||
var weights []hf.File
|
||||
for _, f := range repo.Files {
|
||||
if !strings.Contains(f.Path, "/") && strings.HasSuffix(f.Path, ".safetensors") {
|
||||
weights = append(weights, f)
|
||||
}
|
||||
}
|
||||
sets := hf.Group(weights)
|
||||
if len(sets) == 0 {
|
||||
return hf.FileSet{}, fmt.Errorf("%s has no safetensors weights at its top", repo.ID)
|
||||
}
|
||||
// Sharded "model-00001-of-00004" or a single "model.safetensors"; a
|
||||
// repository with a second, consolidated copy keeps it under another name.
|
||||
set := sets[0]
|
||||
for _, st := range sets {
|
||||
if strings.HasPrefix(st.Name, "model") {
|
||||
set = st
|
||||
break
|
||||
}
|
||||
}
|
||||
companions := hf.Companions(repo.Files)
|
||||
if !slices.ContainsFunc(companions, func(f hf.File) bool { return f.Path == "config.json" }) {
|
||||
return hf.FileSet{}, fmt.Errorf("%s has no config.json, which vLLM reads the model's shape from", repo.ID)
|
||||
}
|
||||
for _, f := range companions {
|
||||
set.Files = append(set.Files, f)
|
||||
set.Bytes += f.Bytes
|
||||
}
|
||||
return set, nil
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/binary"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -223,3 +224,68 @@ func TestARepositoryThatIsNotOneIsRefused(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAVisionModelIsServedWithItsProjectorBesideIt(t *testing.T) {
|
||||
files := []hf.File{
|
||||
{Path: "gemma-3-4b-it-Q4_K_M.gguf", Bytes: 2 << 30, SHA256: "aa"},
|
||||
{Path: "mmproj-gemma-3-4b-it-f16.gguf", Bytes: 800 << 20, SHA256: "bb"},
|
||||
}
|
||||
s, m, c, _ := setup(t, files, map[string]string{"gemma-3-4b-it-Q4_K_M.gguf": "aa", "mmproj-gemma-3-4b-it-f16.gguf": "bb"})
|
||||
job, err := s.Start(Request{Repo: "google/gemma-3-4b-it-GGUF", Quant: "Q4_K_M",
|
||||
Projector: "mmproj-gemma-3-4b-it-f16.gguf", Replicas: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if j := wait(t, s, job.ID); j.State != Done {
|
||||
t.Fatalf("%s %+v", j.Error, j.Steps)
|
||||
}
|
||||
if len(m.got) != 2 {
|
||||
t.Fatalf("fetched %+v", m.got)
|
||||
}
|
||||
got := c.added[0]
|
||||
if !strings.HasSuffix(got.Source, "/") || !got.Capabilities.Vision || len(got.Origin.Files) != 2 {
|
||||
t.Errorf("%s vision=%v origin=%+v", got.Source, got.Capabilities.Vision, got.Origin)
|
||||
}
|
||||
// A projector the repository does not have is named with those it has.
|
||||
job, _ = s.Start(Request{Repo: "google/gemma-3-4b-it-GGUF", Quant: "Q4_K_M", Projector: "mmproj-x.gguf", Name: "g2"})
|
||||
if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "mmproj-gemma-3-4b-it-f16.gguf") {
|
||||
t.Errorf("%+v", j)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOriginalWeightsAreServedOnVLLMWithTheirConfigAndTokenizer(t *testing.T) {
|
||||
files := []hf.File{
|
||||
{Path: "model-00001-of-00002.safetensors", Bytes: 4 << 30, SHA256: "a1"},
|
||||
{Path: "model-00002-of-00002.safetensors", Bytes: 3 << 30, SHA256: "a2"},
|
||||
{Path: "model.safetensors.index.json", Bytes: 30000},
|
||||
{Path: "config.json", Bytes: 700},
|
||||
{Path: "tokenizer.json", Bytes: 7 << 20},
|
||||
{Path: "tokenizer_config.json", Bytes: 7000},
|
||||
{Path: "README.md", Bytes: 9000},
|
||||
{Path: "original/consolidated.safetensors", Bytes: 7 << 30},
|
||||
}
|
||||
s, m, c, _ := setup(t, files, map[string]string{"model-00001-of-00002.safetensors": "a1", "model-00002-of-00002.safetensors": "a2"})
|
||||
if _, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors"}); err == nil {
|
||||
t.Error("safetensors with no GPU class was taken")
|
||||
}
|
||||
job, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors", GPUClass: "l40s-48g", Replicas: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if j := wait(t, s, job.ID); j.State != Done {
|
||||
t.Fatalf("%s %+v", j.Error, j.Steps)
|
||||
}
|
||||
var got []string
|
||||
for _, r := range m.got {
|
||||
got = append(got, r.File)
|
||||
}
|
||||
if len(got) != 6 || slices.Contains(got, "README.md") || slices.Contains(got, "consolidated.safetensors") {
|
||||
t.Errorf("fetched %v", got)
|
||||
}
|
||||
mod := c.added[0]
|
||||
p := mod.Profiles[0]
|
||||
if !strings.HasSuffix(mod.Source, "/") || p.Engine != "vllm" || p.GPUClass != "l40s-48g" || p.TP != 1 ||
|
||||
!slices.Contains(p.Args, "--max-model-len") {
|
||||
t.Errorf("%s %+v", mod.Source, p)
|
||||
}
|
||||
}
|
||||
@@ -509,9 +509,9 @@ func (s *Server) runUpdate(j *Job) {
|
||||
// 3. Whether the nodes load it, and whether it still fits what is
|
||||
// reserved for it.
|
||||
s.step(j, "check", Running, "reading its header from the mirror")
|
||||
detail := "a set of shards; not read"
|
||||
if !strings.HasSuffix(source, "/") {
|
||||
v := gguf.Check(ctx, source, s.deps.LlamaVersion)
|
||||
detail := "no header to read"
|
||||
if u := headerURL(source, set); u != "" {
|
||||
v := gguf.Check(ctx, u, s.deps.LlamaVersion)
|
||||
if !v.Servable {
|
||||
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason))
|
||||
return
|
||||
|
||||
Reference in new issue
Block a user