Ilker ManapandClaude Opus 5.5
44425193bd
A model served from the registry records its origin: repository, commit and each file's checksum; one served before is worked out from the mirror's record of where it fetched the file. Every mirror.update_check (6h) the control plane asks whether those files changed there. ctl model updates lists it, ctl model update fetches beside the old files, checks, swaps and replaces replicas one at a time, ctl model rollback goes back. The console shows it on Models and offers the same. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
87 lines
3.2 KiB
Go
87 lines
3.2 KiB
Go
package cli
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
|
)
|
|
|
|
func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
|
var (
|
|
req modelserve.Request
|
|
onDemand bool
|
|
)
|
|
c := &cobra.Command{
|
|
Use: "serve OWNER/REPO",
|
|
Short: "Take a model from the registry to a replica that answers, in one command",
|
|
Long: "Picks the GGUF file (--quant, Q4_K_M by default, or --file), has the mirror\n" +
|
|
"fetch it and checks what arrived against the registry's checksum, checks\n" +
|
|
"that the nodes' llama.cpp can load it and sizes its memory from its own\n" +
|
|
"header, adds it to the catalog, waits for a replica, and asks it to say\n" +
|
|
"hello. Each step is shown as it happens; the first that fails stops it.\n\n" +
|
|
" llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" +
|
|
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" +
|
|
" llmgw ctl model serve owner/big-GGUF --on-demand",
|
|
Args: cobra.ExactArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
req.Repo = args[0]
|
|
if onDemand {
|
|
req.Replicas = 0
|
|
}
|
|
body, _ := json.Marshal(req)
|
|
var job modelserve.Job
|
|
if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
|
|
return err
|
|
}
|
|
return followJob(cmd, client, job, "is served")
|
|
},
|
|
}
|
|
f := c.Flags()
|
|
f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)")
|
|
f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant")
|
|
f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
|
|
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
|
|
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
|
|
f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run")
|
|
f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for")
|
|
f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots")
|
|
f.IntVar(&req.CPUCores, "cores", 0, "CPU cores a replica computes on; 0 takes half the smallest node it fits")
|
|
f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use")
|
|
return c
|
|
}
|
|
|
|
// followJob shows a job's steps as they happen, until it ends.
|
|
func followJob(cmd *cobra.Command, client func() *ctlClient, job modelserve.Job, doneSays string) error {
|
|
out := cmd.OutOrStdout()
|
|
fmt.Fprintf(out, "job %s\n", job.ID)
|
|
shown := map[string]string{}
|
|
for {
|
|
for _, s := range job.Steps {
|
|
line := s.State + " " + s.Detail
|
|
if shown[s.Name] == line {
|
|
continue
|
|
}
|
|
shown[s.Name] = line
|
|
mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok",
|
|
modelserve.Failed: "!!", modelserve.Skipped: "--"}[s.State]
|
|
fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail)
|
|
}
|
|
switch job.State {
|
|
case modelserve.Done:
|
|
fmt.Fprintf(out, "\n%s %s.\n", job.Model, doneSays)
|
|
return nil
|
|
case modelserve.Failed:
|
|
return errors.New(job.Error)
|
|
}
|
|
time.Sleep(2 * time.Second)
|
|
if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|