Serve vision models with their projector and original weights on vLLM; group the mirror by version
model serve takes --projector, fetched beside the weights and loaded by llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp, which fetches the weights with their config and tokenizer and runs them on vLLM. The console offers both, F switching between GGUF and original weights. The mirror view is one row per version with its files on Enter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
7aefb236d6
commit
a16097cfe8
12 files changed
+509
-81
No files matched your search
@@ -509,9 +509,9 @@ func (s *Server) runUpdate(j *Job) {
|
||||
// 3. Whether the nodes load it, and whether it still fits what is
|
||||
// reserved for it.
|
||||
s.step(j, "check", Running, "reading its header from the mirror")
|
||||
detail := "a set of shards; not read"
|
||||
if !strings.HasSuffix(source, "/") {
|
||||
v := gguf.Check(ctx, source, s.deps.LlamaVersion)
|
||||
detail := "no header to read"
|
||||
if u := headerURL(source, set); u != "" {
|
||||
v := gguf.Check(ctx, u, s.deps.LlamaVersion)
|
||||
if !v.Servable {
|
||||
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason))
|
||||
return
|
||||
|
||||
Reference in new issue
Block a user