Serve a model from the registry with one command
Adding a model took four steps and a YAML file — plan it, fetch it into the mirror, write its catalog entry with a memory figure somebody estimated, scale it — and whether it would load was found out at the end. `llmgw ctl model serve OWNER/REPO` does it as one job in the control plane, where the registry, the mirror, the catalog and the fleet already are: - the registry: the GGUF file set matching --quant (Q4_K_M by default) or --file, leaving out vision projectors and importance matrices, and naming what the repository does have when nothing matches; gated repositories are refused; - the mirror: fetched at the repository's revision, with progress, and each file's sha256 compared with what the registry published — the mirror is the one place a corrupted download is caught once rather than loaded on every node; - the check: whether the nodes' llama.cpp loads the architecture, and the memory it needs from its own header; - the catalog, then a wait for a replica that stops early, with the reason, when the model is parked or fits on no node; - a smoke test: one short request to the replica, with its engine key, and the answer and tokens per second. ctl starts the job and prints each step as it changes; a download of hours does not depend on the terminal. --on-demand adds the model without starting a replica. Jobs are kept in memory until the control plane restarts. The catalog name a repository suggests moved into the registry package, so the mirror page and this agree on it. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
116e75f8cb
commit
bd12b0909a
9 files changed
+946
-8
No files matched your search
+20
-1
@@ -5,6 +5,7 @@ import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
@@ -171,8 +172,16 @@ func newServerCmd() *cobra.Command {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Kept past this block: serving a model from the registry needs
|
||||
// both the admin API and the mirror, and the mirror is set up
|
||||
// below.
|
||||
var (
|
||||
adminAPI *adminapi.API
|
||||
catalogSvc *modelsvc.Service
|
||||
)
|
||||
if admin != nil {
|
||||
api := adminapi.New(admin)
|
||||
adminAPI = api
|
||||
api.SetReleases(cfg.Releases)
|
||||
// Catalog changes need somewhere to persist, so this is wired
|
||||
// only when there is a store. Without one, the endpoints say so
|
||||
@@ -190,6 +199,7 @@ func newServerCmd() *cobra.Command {
|
||||
admin.ReplaceCatalog(ctx, built)
|
||||
}
|
||||
api.SetModels(svc)
|
||||
catalogSvc = svc
|
||||
// And the mirror page, so a model that has just been
|
||||
// fetched can be served without writing YAML by hand.
|
||||
rt.SetCatalogAdmin(svc)
|
||||
@@ -259,7 +269,16 @@ func newServerCmd() *cobra.Command {
|
||||
// Browsing the registry is what turns "paste a resolve URL"
|
||||
// into "find the model". It needs a way out to the internet
|
||||
// from this machine, which the mirror's own fetching does not.
|
||||
rt.SetRegistry(hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout))
|
||||
registry := hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout)
|
||||
rt.SetRegistry(registry)
|
||||
// One command from a registry repository to a replica that
|
||||
// answers: `llmgw ctl model serve`.
|
||||
if adminAPI != nil && catalogSvc != nil {
|
||||
adminAPI.SetServe(modelserve.New(modelserve.Deps{
|
||||
Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin,
|
||||
LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log,
|
||||
}))
|
||||
}
|
||||
log.Info("model mirror configured",
|
||||
"api", cfg.Mirror.API, "mtls", cfg.Mirror.TLS.Enabled())
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user