Ilker ManapandClaude Opus 5.5
bd12b0909a
Adding a model took four steps and a YAML file — plan it, fetch it into the mirror, write its catalog entry with a memory figure somebody estimated, scale it — and whether it would load was found out at the end. `llmgw ctl model serve OWNER/REPO` does it as one job in the control plane, where the registry, the mirror, the catalog and the fleet already are: - the registry: the GGUF file set matching --quant (Q4_K_M by default) or --file, leaving out vision projectors and importance matrices, and naming what the repository does have when nothing matches; gated repositories are refused; - the mirror: fetched at the repository's revision, with progress, and each file's sha256 compared with what the registry published — the mirror is the one place a corrupted download is caught once rather than loaded on every node; - the check: whether the nodes' llama.cpp loads the architecture, and the memory it needs from its own header; - the catalog, then a wait for a replica that stops early, with the reason, when the model is parked or fits on no node; - a smoke test: one short request to the replica, with its engine key, and the answer and tokens per second. ctl starts the job and prints each step as it changes; a download of hours does not depend on the terminal. --on-demand adds the model without starting a replica. Jobs are kept in memory until the control plane restarts. The catalog name a repository suggests moved into the registry package, so the mirror page and this agree on it. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
81 lines
2.9 KiB
Go
81 lines
2.9 KiB
Go
package cli
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
|
)
|
|
|
|
func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
|
var (
|
|
req modelserve.Request
|
|
onDemand bool
|
|
)
|
|
c := &cobra.Command{
|
|
Use: "serve OWNER/REPO",
|
|
Short: "Take a model from the registry to a replica that answers, in one command",
|
|
Long: "Picks the GGUF file (--quant, Q4_K_M by default, or --file), has the mirror\n" +
|
|
"fetch it and checks what arrived against the registry's checksum, checks\n" +
|
|
"that the nodes' llama.cpp can load it and sizes its memory from its own\n" +
|
|
"header, adds it to the catalog, waits for a replica, and asks it to say\n" +
|
|
"hello. Each step is shown as it happens; the first that fails stops it.\n\n" +
|
|
" llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" +
|
|
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" +
|
|
" llmgw ctl model serve owner/big-GGUF --on-demand",
|
|
Args: cobra.ExactArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
req.Repo = args[0]
|
|
if onDemand {
|
|
req.Replicas = 0
|
|
}
|
|
body, _ := json.Marshal(req)
|
|
var job modelserve.Job
|
|
if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
|
|
return err
|
|
}
|
|
out := cmd.OutOrStdout()
|
|
fmt.Fprintf(out, "job %s\n", job.ID)
|
|
shown := map[string]string{}
|
|
for {
|
|
for _, s := range job.Steps {
|
|
line := s.State + " " + s.Detail
|
|
if shown[s.Name] == line {
|
|
continue
|
|
}
|
|
shown[s.Name] = line
|
|
mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok",
|
|
modelserve.Failed: "!!"}[s.State]
|
|
fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail)
|
|
}
|
|
switch job.State {
|
|
case modelserve.Done:
|
|
fmt.Fprintf(out, "\n%s is served.\n", job.Model)
|
|
return nil
|
|
case modelserve.Failed:
|
|
return errors.New(job.Error)
|
|
}
|
|
time.Sleep(2 * time.Second)
|
|
if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
},
|
|
}
|
|
f := c.Flags()
|
|
f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)")
|
|
f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant")
|
|
f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
|
|
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
|
|
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
|
|
f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run")
|
|
f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for")
|
|
f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots")
|
|
f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use")
|
|
return c
|
|
}
|