Serve vision models with their projector and original weights on vLLM; group the mirror by version
model serve takes --projector, fetched beside the weights and loaded by llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp, which fetches the weights with their config and tokenizer and runs them on vLLM. The console offers both, F switching between GGUF and original weights. The mirror view is one row per version with its files on Enter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
7aefb236d6
commit
a16097cfe8
12 files changed
+509
-81
No files matched your search
+11
-4
@@ -68,6 +68,10 @@ type Model struct {
|
||||
// GGUF is what the registry read from the repository's weights, absent
|
||||
// for one that has none.
|
||||
GGUF *GGUFInfo `json:"gguf"`
|
||||
// Safetensors is the registry's summary of original weights.
|
||||
Safetensors *struct {
|
||||
Total int64 `json:"total"`
|
||||
} `json:"safetensors,omitempty"`
|
||||
// Siblings are the repository's files, when asked for with WithFiles.
|
||||
Siblings []struct {
|
||||
RFilename string `json:"rfilename"`
|
||||
@@ -84,10 +88,13 @@ type GGUFInfo struct {
|
||||
|
||||
// Params is the parameter count, 0 when the registry does not know it.
|
||||
func (m Model) Params() int64 {
|
||||
if m.GGUF == nil {
|
||||
return 0
|
||||
switch {
|
||||
case m.GGUF != nil:
|
||||
return m.GGUF.Total
|
||||
case m.Safetensors != nil:
|
||||
return m.Safetensors.Total
|
||||
}
|
||||
return m.GGUF.Total
|
||||
return 0
|
||||
}
|
||||
|
||||
// Architecture is the GGUF architecture, empty when unknown.
|
||||
@@ -172,7 +179,7 @@ type Query struct {
|
||||
// exploreFields is everything a listing shows. Asking for any field with
|
||||
// expand returns only the fields asked for, so all of them are named.
|
||||
var exploreFields = []string{"downloads", "likes", "pipeline_tag", "tags", "lastModified",
|
||||
"createdAt", "gated", "private", "trendingScore", "gguf"}
|
||||
"createdAt", "gated", "private", "trendingScore", "gguf", "safetensors"}
|
||||
|
||||
// Explore lists repositories: the trending ones, the newest, or a search.
|
||||
//
|
||||
|
||||
Reference in new issue
Block a user