Serve vision models with their projector and original weights on vLLM; group the mirror by version

model serve takes --projector, fetched beside the weights and loaded by
llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp,
which fetches the weights with their config and tokenizer and runs them on
vLLM. The console offers both, F switching between GGUF and original
weights. The mirror view is one row per version with its files on Enter.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 00:42:59 +02:00
1 parent 7aefb236d6
commit a16097cfe8
12 files changed
+509 -81

No files matched your search

+18 -1
View File
@@ -8,6 +8,7 @@ import (
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
)
@@ -24,15 +25,27 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
"that the nodes' llama.cpp can load it and sizes its memory from its own\n" +
"header, adds it to the catalog, waits for a replica, and asks it to say\n" +
"hello. Each step is shown as it happens; the first that fails stops it.\n\n" +
"A vision model's projector (--projector) is fetched beside it and loaded with\n" +
"it. --format safetensors serves the original weights instead, with their\n" +
"config.json and tokenizer, on vLLM across --tp GPUs of --gpu-class.\n\n" +
" llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" +
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" +
" llmgw ctl model serve owner/big-GGUF --on-demand",
" llmgw ctl model serve owner/big-GGUF --on-demand\n" +
" llmgw ctl model serve google/gemma-3-4b-it-GGUF --projector mmproj-model-f16.gguf\n" +
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct --format safetensors --gpu-class l40s-48g",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
req.Repo = args[0]
if onDemand {
req.Replicas = 0
}
// One kind of GPU in the fleet: that one, unless told.
if req.Format == "safetensors" && req.GPUClass == "" {
var sum adminapi.Summary
if err := client().get(cmd.Context(), "/api/v1/summary", &sum); err == nil && len(sum.GPUs) == 1 {
req.GPUClass = sum.GPUs[0].Class
}
}
body, _ := json.Marshal(req)
var job modelserve.Job
if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
@@ -44,6 +57,10 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
f := c.Flags()
f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)")
f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant")
f.StringVar(&req.Projector, "projector", "", "a vision model's mmproj file, fetched and loaded beside it")
f.StringVar(&req.Format, "format", "gguf", "gguf, run by llama.cpp, or safetensors: the original weights, run by vLLM on GPUs")
f.StringVar(&req.GPUClass, "gpu-class", "", "for safetensors: the GPUs to run on (default: the fleet's one class)")
f.IntVar(&req.TP, "tp", 1, "for safetensors: how many GPUs one replica spans")
f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")