Plan a repository's files against the fleet, and serve the one that suits it the way it fits
The control plane reads the model's shape from one GGUF header or its config.json and lists, for every file, the best way it runs here: one card, several of a node's (by layer for llama.cpp, tensor-parallel for vLLM), several nodes' over one fabric (vLLM, pipeline-parallel between them), or a CPU node; what each card holds and whether it is free now. ctl model plan prints it; model serve takes its file and way unless told, gives a GGUF on GPUs a GPU profile, and with --fallbacks adds the next best ways. The console shows it per file and asks for the way and the fallback. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
630d2788c0
commit
54db7aecb7
15 files changed
+1224
-158
No files matched your search
+16
-13
@@ -8,7 +8,6 @@ import (
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
)
|
||||
|
||||
@@ -25,27 +24,26 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
"that the nodes' llama.cpp can load it and sizes its memory from its own\n" +
|
||||
"header, adds it to the catalog, waits for a replica, and asks it to say\n" +
|
||||
"hello. Each step is shown as it happens; the first that fails stops it.\n\n" +
|
||||
"Without --quant or --file, the file is the one that suits the cluster best,\n" +
|
||||
"and where it runs is the plan's best way for it: on one card, on several of\n" +
|
||||
"a node's, over several nodes on a fabric, or on a CPU node (`llmgw ctl model\n" +
|
||||
"plan REPO` lists them). --cpu or --gpu-class with --gpus and --nodes say it\n" +
|
||||
"instead; --fallbacks adds the next best ways for when the first is busy.\n\n" +
|
||||
"A vision model's projector (--projector) is fetched beside it and loaded with\n" +
|
||||
"it. --format safetensors serves the original weights instead, with their\n" +
|
||||
"config.json and tokenizer, on vLLM across --tp GPUs of --gpu-class.\n\n" +
|
||||
"config.json and tokenizer, on vLLM.\n\n" +
|
||||
" llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" +
|
||||
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" +
|
||||
" llmgw ctl model serve owner/big-GGUF --on-demand\n" +
|
||||
" llmgw ctl model serve google/gemma-3-4b-it-GGUF --projector mmproj-model-f16.gguf\n" +
|
||||
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct --format safetensors --gpu-class l40s-48g",
|
||||
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct --format safetensors --fallbacks\n" +
|
||||
" llmgw ctl model serve meta-llama/Llama-3.1-405B-Instruct --format safetensors --gpu-class h100-80g --gpus 8 --nodes 2",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
req.Repo = args[0]
|
||||
if onDemand {
|
||||
req.Replicas = 0
|
||||
}
|
||||
// One kind of GPU in the fleet: that one, unless told.
|
||||
if req.Format == "safetensors" && req.GPUClass == "" {
|
||||
var sum adminapi.Summary
|
||||
if err := client().get(cmd.Context(), "/api/v1/summary", &sum); err == nil && len(sum.GPUs) == 1 {
|
||||
req.GPUClass = sum.GPUs[0].Class
|
||||
}
|
||||
}
|
||||
body, _ := json.Marshal(req)
|
||||
var job modelserve.Job
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
|
||||
@@ -55,12 +53,17 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
},
|
||||
}
|
||||
f := c.Flags()
|
||||
f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)")
|
||||
f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default: the one that suits the cluster best)")
|
||||
f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant")
|
||||
f.StringVar(&req.Projector, "projector", "", "a vision model's mmproj file, fetched and loaded beside it")
|
||||
f.StringVar(&req.Format, "format", "gguf", "gguf, run by llama.cpp, or safetensors: the original weights, run by vLLM on GPUs")
|
||||
f.StringVar(&req.GPUClass, "gpu-class", "", "for safetensors: the GPUs to run on (default: the fleet's one class)")
|
||||
f.IntVar(&req.TP, "tp", 1, "for safetensors: how many GPUs one replica spans")
|
||||
f.BoolVar(&req.CPU, "cpu", false, "run it on the CPU, whatever the plan says")
|
||||
f.StringVar(&req.GPUClass, "gpu-class", "", "run it on GPUs of this class, whatever the plan says")
|
||||
f.IntVar(&req.GPUsPerNode, "gpus", 0, "with --gpu-class: the cards one replica takes on each node (default 1)")
|
||||
f.IntVar(&req.Nodes, "nodes", 0, "with --gpu-class: the nodes one replica spans, over a fabric (vLLM only)")
|
||||
f.IntVar(&req.TP, "tp", 0, "the older name of --gpus")
|
||||
_ = f.MarkHidden("tp")
|
||||
f.BoolVar(&req.Fallbacks, "fallbacks", false, "also add the next best ways to run it, used when the first is busy")
|
||||
f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
|
||||
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
|
||||
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
|
||||
|
||||
Reference in new issue
Block a user