Files
llmgw/internal/cli/model_serve.go
T
Ilker ManapandClaude Opus 5.5 54db7aecb7 Plan a repository's files against the fleet, and serve the one that suits it the way it fits
The control plane reads the model's shape from one GGUF header or its
config.json and lists, for every file, the best way it runs here: one card,
several of a node's (by layer for llama.cpp, tensor-parallel for vLLM),
several nodes' over one fabric (vLLM, pipeline-parallel between them), or a
CPU node; what each card holds and whether it is free now. ctl model plan
prints it; model serve takes its file and way unless told, gives a GGUF on
GPUs a GPU profile, and with --fallbacks adds the next best ways. The
console shows it per file and asks for the way and the fallback.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-04 01:03:48 +02:00

107 lines
5.0 KiB
Go

package cli
import (
"encoding/json"
"errors"
"fmt"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
)
func newModelServeCmd(client func() *ctlClient) *cobra.Command {
var (
req modelserve.Request
onDemand bool
)
c := &cobra.Command{
Use: "serve OWNER/REPO",
Short: "Take a model from the registry to a replica that answers, in one command",
Long: "Picks the GGUF file (--quant, Q4_K_M by default, or --file), has the mirror\n" +
"fetch it and checks what arrived against the registry's checksum, checks\n" +
"that the nodes' llama.cpp can load it and sizes its memory from its own\n" +
"header, adds it to the catalog, waits for a replica, and asks it to say\n" +
"hello. Each step is shown as it happens; the first that fails stops it.\n\n" +
"Without --quant or --file, the file is the one that suits the cluster best,\n" +
"and where it runs is the plan's best way for it: on one card, on several of\n" +
"a node's, over several nodes on a fabric, or on a CPU node (`llmgw ctl model\n" +
"plan REPO` lists them). --cpu or --gpu-class with --gpus and --nodes say it\n" +
"instead; --fallbacks adds the next best ways for when the first is busy.\n\n" +
"A vision model's projector (--projector) is fetched beside it and loaded with\n" +
"it. --format safetensors serves the original weights instead, with their\n" +
"config.json and tokenizer, on vLLM.\n\n" +
" llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" +
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" +
" llmgw ctl model serve owner/big-GGUF --on-demand\n" +
" llmgw ctl model serve google/gemma-3-4b-it-GGUF --projector mmproj-model-f16.gguf\n" +
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct --format safetensors --fallbacks\n" +
" llmgw ctl model serve meta-llama/Llama-3.1-405B-Instruct --format safetensors --gpu-class h100-80g --gpus 8 --nodes 2",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
req.Repo = args[0]
if onDemand {
req.Replicas = 0
}
body, _ := json.Marshal(req)
var job modelserve.Job
if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
return err
}
return followJob(cmd, client, job, "is served")
},
}
f := c.Flags()
f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default: the one that suits the cluster best)")
f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant")
f.StringVar(&req.Projector, "projector", "", "a vision model's mmproj file, fetched and loaded beside it")
f.StringVar(&req.Format, "format", "gguf", "gguf, run by llama.cpp, or safetensors: the original weights, run by vLLM on GPUs")
f.BoolVar(&req.CPU, "cpu", false, "run it on the CPU, whatever the plan says")
f.StringVar(&req.GPUClass, "gpu-class", "", "run it on GPUs of this class, whatever the plan says")
f.IntVar(&req.GPUsPerNode, "gpus", 0, "with --gpu-class: the cards one replica takes on each node (default 1)")
f.IntVar(&req.Nodes, "nodes", 0, "with --gpu-class: the nodes one replica spans, over a fabric (vLLM only)")
f.IntVar(&req.TP, "tp", 0, "the older name of --gpus")
_ = f.MarkHidden("tp")
f.BoolVar(&req.Fallbacks, "fallbacks", false, "also add the next best ways to run it, used when the first is busy")
f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run")
f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for")
f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots")
f.IntVar(&req.CPUCores, "cores", 0, "CPU cores a replica computes on; 0 takes half the smallest node it fits")
f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use")
return c
}
// followJob shows a job's steps as they happen, until it ends.
func followJob(cmd *cobra.Command, client func() *ctlClient, job modelserve.Job, doneSays string) error {
out := cmd.OutOrStdout()
fmt.Fprintf(out, "job %s\n", job.ID)
shown := map[string]string{}
for {
for _, s := range job.Steps {
line := s.State + " " + s.Detail
if shown[s.Name] == line {
continue
}
shown[s.Name] = line
mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok",
modelserve.Failed: "!!", modelserve.Skipped: "--"}[s.State]
fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail)
}
switch job.State {
case modelserve.Done:
fmt.Fprintf(out, "\n%s %s.\n", job.Model, doneSays)
return nil
case modelserve.Failed:
return errors.New(job.Error)
}
time.Sleep(2 * time.Second)
if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil {
return err
}
}
}