Track the registry's new weights for served models, and move a model to them or back
A model served from the registry records its origin: repository, commit and each file's checksum; one served before is worked out from the mirror's record of where it fetched the file. Every mirror.update_check (6h) the control plane asks whether those files changed there. ctl model updates lists it, ctl model update fetches beside the old files, checks, swaps and replaces replicas one at a time, ctl model rollback goes back. The console shows it on Models and offers the same. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a96b1a54b2
commit
44425193bd
15 files changed
+1393
-33
No files matched your search
+15
-2
@@ -128,6 +128,7 @@ func newServerCmd() *cobra.Command {
|
||||
var agentHandler http.Handler
|
||||
var agentPath string
|
||||
var admin *control.Controller
|
||||
updateMetrics := &modelserve.UpdateCollector{}
|
||||
// The store's cluster, when it is one; read by the metrics and
|
||||
// by `ctl status`.
|
||||
var storeHA *pgha.Collector
|
||||
@@ -183,6 +184,11 @@ func newServerCmd() *cobra.Command {
|
||||
if err := rt.RegisterMetrics(fleetmetrics.New(ctl)); err != nil {
|
||||
return err
|
||||
}
|
||||
// Whether models' repositories have new weights; filled in
|
||||
// once the mirror is set up, below.
|
||||
if err := rt.RegisterMetrics(updateMetrics); err != nil {
|
||||
return err
|
||||
}
|
||||
// A store of several hosts is a replicated cluster: what state
|
||||
// it is in is published beside everything else, read from its
|
||||
// Patroni. One host is a single PostgreSQL, which has none.
|
||||
@@ -321,10 +327,17 @@ func newServerCmd() *cobra.Command {
|
||||
// One command from a registry repository to a replica that
|
||||
// answers: `llmgw ctl model serve`.
|
||||
if adminAPI != nil && catalogSvc != nil {
|
||||
adminAPI.SetServe(modelserve.New(modelserve.Deps{
|
||||
serve := modelserve.New(modelserve.Deps{
|
||||
Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin,
|
||||
LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log,
|
||||
}))
|
||||
// Updates: the catalog in force, the service that
|
||||
// changes it, and what replaces replicas.
|
||||
Current: func() []catalog.Model { return admin.Catalog().Models() },
|
||||
Models: catalogSvc, Restart: admin.Reconciler().Restart,
|
||||
})
|
||||
adminAPI.SetServe(serve)
|
||||
updateMetrics.Set(serve)
|
||||
go serve.WatchUpdates(ctx, cfg.Mirror.UpdateCheck)
|
||||
}
|
||||
log.Info("model mirror configured",
|
||||
"api", cfg.Mirror.API, "mtls", cfg.Mirror.TLS.Enabled())
|
||||
|
||||
Reference in new issue
Block a user