Ilker ManapandClaude Opus 5.5
990d82d801
A profile's cpu_cores is reserved out of its node's CPUs by placement, as memory is, and becomes the engine's thread count. Agents report their CPU count in registration and heartbeats. New CPU models must declare memory; model views show reserved against measured memory with warnings, and ctl model resources sets both for a stored model. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
82 lines
3.0 KiB
Go
82 lines
3.0 KiB
Go
package cli
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
|
)
|
|
|
|
func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
|
var (
|
|
req modelserve.Request
|
|
onDemand bool
|
|
)
|
|
c := &cobra.Command{
|
|
Use: "serve OWNER/REPO",
|
|
Short: "Take a model from the registry to a replica that answers, in one command",
|
|
Long: "Picks the GGUF file (--quant, Q4_K_M by default, or --file), has the mirror\n" +
|
|
"fetch it and checks what arrived against the registry's checksum, checks\n" +
|
|
"that the nodes' llama.cpp can load it and sizes its memory from its own\n" +
|
|
"header, adds it to the catalog, waits for a replica, and asks it to say\n" +
|
|
"hello. Each step is shown as it happens; the first that fails stops it.\n\n" +
|
|
" llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" +
|
|
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" +
|
|
" llmgw ctl model serve owner/big-GGUF --on-demand",
|
|
Args: cobra.ExactArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
req.Repo = args[0]
|
|
if onDemand {
|
|
req.Replicas = 0
|
|
}
|
|
body, _ := json.Marshal(req)
|
|
var job modelserve.Job
|
|
if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
|
|
return err
|
|
}
|
|
out := cmd.OutOrStdout()
|
|
fmt.Fprintf(out, "job %s\n", job.ID)
|
|
shown := map[string]string{}
|
|
for {
|
|
for _, s := range job.Steps {
|
|
line := s.State + " " + s.Detail
|
|
if shown[s.Name] == line {
|
|
continue
|
|
}
|
|
shown[s.Name] = line
|
|
mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok",
|
|
modelserve.Failed: "!!"}[s.State]
|
|
fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail)
|
|
}
|
|
switch job.State {
|
|
case modelserve.Done:
|
|
fmt.Fprintf(out, "\n%s is served.\n", job.Model)
|
|
return nil
|
|
case modelserve.Failed:
|
|
return errors.New(job.Error)
|
|
}
|
|
time.Sleep(2 * time.Second)
|
|
if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
},
|
|
}
|
|
f := c.Flags()
|
|
f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)")
|
|
f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant")
|
|
f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
|
|
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
|
|
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
|
|
f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run")
|
|
f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for")
|
|
f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots")
|
|
f.IntVar(&req.CPUCores, "cores", 0, "CPU cores a replica computes on; 0 takes half the smallest node it fits")
|
|
f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use")
|
|
return c
|
|
}
|