Serve a model from the registry with one command
Adding a model took four steps and a YAML file — plan it, fetch it into the mirror, write its catalog entry with a memory figure somebody estimated, scale it — and whether it would load was found out at the end. `llmgw ctl model serve OWNER/REPO` does it as one job in the control plane, where the registry, the mirror, the catalog and the fleet already are: - the registry: the GGUF file set matching --quant (Q4_K_M by default) or --file, leaving out vision projectors and importance matrices, and naming what the repository does have when nothing matches; gated repositories are refused; - the mirror: fetched at the repository's revision, with progress, and each file's sha256 compared with what the registry published — the mirror is the one place a corrupted download is caught once rather than loaded on every node; - the check: whether the nodes' llama.cpp loads the architecture, and the memory it needs from its own header; - the catalog, then a wait for a replica that stops early, with the reason, when the model is parked or fits on no node; - a smoke test: one short request to the replica, with its engine key, and the answer and tokens per second. ctl starts the job and prints each step as it changes; a download of hours does not depend on the terminal. --on-demand adds the model without starting a replica. Jobs are kept in memory until the control plane restarts. The catalog name a repository suggests moved into the registry package, so the mirror page and this agree on it. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
116e75f8cb
commit
bd12b0909a
9 files changed
+946
-8
No files matched your search
@@ -28,7 +28,8 @@ func newModelCmd(client func() *ctlClient) *cobra.Command {
|
||||
Use: "model",
|
||||
Short: "Add or remove models while the cluster is running",
|
||||
}
|
||||
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client))
|
||||
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
newModelServeCmd(client))
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
)
|
||||
|
||||
func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
var (
|
||||
req modelserve.Request
|
||||
onDemand bool
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "serve OWNER/REPO",
|
||||
Short: "Take a model from the registry to a replica that answers, in one command",
|
||||
Long: "Picks the GGUF file (--quant, Q4_K_M by default, or --file), has the mirror\n" +
|
||||
"fetch it and checks what arrived against the registry's checksum, checks\n" +
|
||||
"that the nodes' llama.cpp can load it and sizes its memory from its own\n" +
|
||||
"header, adds it to the catalog, waits for a replica, and asks it to say\n" +
|
||||
"hello. Each step is shown as it happens; the first that fails stops it.\n\n" +
|
||||
" llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" +
|
||||
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" +
|
||||
" llmgw ctl model serve owner/big-GGUF --on-demand",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
req.Repo = args[0]
|
||||
if onDemand {
|
||||
req.Replicas = 0
|
||||
}
|
||||
body, _ := json.Marshal(req)
|
||||
var job modelserve.Job
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
|
||||
return err
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "job %s\n", job.ID)
|
||||
shown := map[string]string{}
|
||||
for {
|
||||
for _, s := range job.Steps {
|
||||
line := s.State + " " + s.Detail
|
||||
if shown[s.Name] == line {
|
||||
continue
|
||||
}
|
||||
shown[s.Name] = line
|
||||
mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok",
|
||||
modelserve.Failed: "!!"}[s.State]
|
||||
fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail)
|
||||
}
|
||||
switch job.State {
|
||||
case modelserve.Done:
|
||||
fmt.Fprintf(out, "\n%s is served.\n", job.Model)
|
||||
return nil
|
||||
case modelserve.Failed:
|
||||
return errors.New(job.Error)
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
f := c.Flags()
|
||||
f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)")
|
||||
f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant")
|
||||
f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
|
||||
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
|
||||
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
|
||||
f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run")
|
||||
f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for")
|
||||
f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots")
|
||||
f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use")
|
||||
return c
|
||||
}
|
||||
+20
-1
@@ -5,6 +5,7 @@ import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
@@ -171,8 +172,16 @@ func newServerCmd() *cobra.Command {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Kept past this block: serving a model from the registry needs
|
||||
// both the admin API and the mirror, and the mirror is set up
|
||||
// below.
|
||||
var (
|
||||
adminAPI *adminapi.API
|
||||
catalogSvc *modelsvc.Service
|
||||
)
|
||||
if admin != nil {
|
||||
api := adminapi.New(admin)
|
||||
adminAPI = api
|
||||
api.SetReleases(cfg.Releases)
|
||||
// Catalog changes need somewhere to persist, so this is wired
|
||||
// only when there is a store. Without one, the endpoints say so
|
||||
@@ -190,6 +199,7 @@ func newServerCmd() *cobra.Command {
|
||||
admin.ReplaceCatalog(ctx, built)
|
||||
}
|
||||
api.SetModels(svc)
|
||||
catalogSvc = svc
|
||||
// And the mirror page, so a model that has just been
|
||||
// fetched can be served without writing YAML by hand.
|
||||
rt.SetCatalogAdmin(svc)
|
||||
@@ -259,7 +269,16 @@ func newServerCmd() *cobra.Command {
|
||||
// Browsing the registry is what turns "paste a resolve URL"
|
||||
// into "find the model". It needs a way out to the internet
|
||||
// from this machine, which the mirror's own fetching does not.
|
||||
rt.SetRegistry(hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout))
|
||||
registry := hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout)
|
||||
rt.SetRegistry(registry)
|
||||
// One command from a registry repository to a replica that
|
||||
// answers: `llmgw ctl model serve`.
|
||||
if adminAPI != nil && catalogSvc != nil {
|
||||
adminAPI.SetServe(modelserve.New(modelserve.Deps{
|
||||
Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin,
|
||||
LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log,
|
||||
}))
|
||||
}
|
||||
log.Info("model mirror configured",
|
||||
"api", cfg.Mirror.API, "mtls", cfg.Mirror.TLS.Enabled())
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user