Serve vision models with their projector and original weights on vLLM; group the mirror by version

model serve takes --projector, fetched beside the weights and loaded by
llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp,
which fetches the weights with their config and tokenizer and runs them on
vLLM. The console offers both, F switching between GGUF and original
weights. The mirror view is one row per version with its files on Enter.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 00:42:59 +02:00
1 parent 7aefb236d6
commit a16097cfe8
12 files changed
+509 -81

No files matched your search

+3 -3
View File
@@ -509,9 +509,9 @@ func (s *Server) runUpdate(j *Job) {
// 3. Whether the nodes load it, and whether it still fits what is
// reserved for it.
s.step(j, "check", Running, "reading its header from the mirror")
detail := "a set of shards; not read"
if !strings.HasSuffix(source, "/") {
v := gguf.Check(ctx, source, s.deps.LlamaVersion)
detail := "no header to read"
if u := headerURL(source, set); u != "" {
v := gguf.Check(ctx, u, s.deps.LlamaVersion)
if !v.Servable {
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason))
return