Serve vision models with their projector and original weights on vLLM; group the mirror by version

model serve takes --projector, fetched beside the weights and loaded by
llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp,
which fetches the weights with their config and tokenizer and runs them on
vLLM. The console offers both, F switching between GGUF and original
weights. The mirror view is one row per version with its files on Enter.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 00:42:59 +02:00
1 parent 7aefb236d6
commit a16097cfe8
12 files changed
+509 -81

No files matched your search

+11 -4
View File
@@ -68,6 +68,10 @@ type Model struct {
// GGUF is what the registry read from the repository's weights, absent
// for one that has none.
GGUF *GGUFInfo `json:"gguf"`
// Safetensors is the registry's summary of original weights.
Safetensors *struct {
Total int64 `json:"total"`
} `json:"safetensors,omitempty"`
// Siblings are the repository's files, when asked for with WithFiles.
Siblings []struct {
RFilename string `json:"rfilename"`
@@ -84,10 +88,13 @@ type GGUFInfo struct {
// Params is the parameter count, 0 when the registry does not know it.
func (m Model) Params() int64 {
if m.GGUF == nil {
return 0
switch {
case m.GGUF != nil:
return m.GGUF.Total
case m.Safetensors != nil:
return m.Safetensors.Total
}
return m.GGUF.Total
return 0
}
// Architecture is the GGUF architecture, empty when unknown.
@@ -172,7 +179,7 @@ type Query struct {
// exploreFields is everything a listing shows. Asking for any field with
// expand returns only the fields asked for, so all of them are named.
var exploreFields = []string{"downloads", "likes", "pipeline_tag", "tags", "lastModified",
"createdAt", "gated", "private", "trendingScore", "gguf"}
"createdAt", "gated", "private", "trendingScore", "gguf", "safetensors"}
// Explore lists repositories: the trending ones, the newest, or a search.
//