1 parent
7465c24830
commit
d6f8f433f4
24 files changed
+437
-22
No files matched your search
@@ -64,6 +64,8 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
"--embedding, 32 slots of 1024 tokens unless --ctx-pool or --context-length says otherwise")
|
||||
f.StringVar(&req.Format, "format", "gguf", "gguf, run by llama.cpp, or safetensors: the original weights, run by vLLM on GPUs")
|
||||
f.BoolVar(&req.CPU, "cpu", false, "run it on the CPU, whatever the plan says")
|
||||
f.StringVar(&req.Engine, "engine", "", "strata, for a family Strata runs: one card and the experts in RAM; "+
|
||||
"llamacpp to keep it off Strata (default: the plan's best)")
|
||||
f.StringVar(&req.GPUClass, "gpu-class", "", "run it on GPUs of this class, whatever the plan says")
|
||||
f.IntVar(&req.GPUsPerNode, "gpus", 0, "with --gpu-class: the cards one replica takes on each node (default 1)")
|
||||
f.IntVar(&req.Nodes, "nodes", 0, "with --gpu-class: the nodes one replica spans, over a fabric (vLLM only)")
|
||||
|
||||
Reference in new issue
Block a user