Federation: another llmgw cluster as a provider, its models as PEER/MODEL
ctl federation add/list/remove/sync keeps peers in the store (migration 0029); their /v1/models are offered here as external models named PEER/MODEL and kept in step, a model still named by a fallback kept withdrawn. Peers are checked at /ready, timed, and taken out of routing while down. Requests to a peer carry x-llmgw-hop and are refused (508) when they come back to a cluster they passed through; residency and no_store keep requests off peers; X-Request-Id and, with --send-tags, attribution tags go along. ctl model fallbacks sets overflow to a peer. Console view, metrics, admin API routes and docs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
40e9fd2ab1
33 files changed
+2944
-23
No files matched your search
@@ -51,6 +51,9 @@ type Service struct {
|
||||
// SetCheck.
|
||||
check func(context.Context, catalog.Model) error
|
||||
|
||||
// federated are the models federated clusters offer; see SetFederated.
|
||||
federated func() []catalog.Model
|
||||
|
||||
// mu serialises rebuilds. They are rare — a person adding a model — and two
|
||||
// at once could apply catalogs in either order, leaving the one in force
|
||||
// disagreeing with the database that outlives it.
|
||||
@@ -128,7 +131,7 @@ func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := append([]catalog.Model{}, s.declared...)
|
||||
next := s.base(stored)
|
||||
for _, e := range stored {
|
||||
if e.Name == m.Name {
|
||||
continue // this call replaces it
|
||||
@@ -174,7 +177,7 @@ func (s *Service) Remove(ctx context.Context, name string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := append([]catalog.Model{}, s.declared...)
|
||||
next := s.base(stored)
|
||||
found := false
|
||||
for _, e := range stored {
|
||||
if e.Name == name {
|
||||
@@ -371,7 +374,7 @@ func (s *Service) update(ctx context.Context, name, by, what string, change func
|
||||
if err != nil {
|
||||
return catalog.Model{}, err
|
||||
}
|
||||
next := append([]catalog.Model{}, s.declared...)
|
||||
next := s.base(stored)
|
||||
found := false
|
||||
for _, e := range stored {
|
||||
if e.Name == name {
|
||||
@@ -416,12 +419,19 @@ func (s *Service) Load(ctx context.Context) (*catalog.Catalog, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
return s.load(ctx)
|
||||
}
|
||||
|
||||
// load is Load, with the lock held.
|
||||
func (s *Service) load(ctx context.Context) (*catalog.Catalog, error) {
|
||||
stored, err := s.storedModels(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
next := append([]catalog.Model{}, s.declared...)
|
||||
// The federated models before the stored ones, so a stored model that
|
||||
// falls back to a peer's is checked against a catalog that has it.
|
||||
next := s.base(stored)
|
||||
for _, m := range stored {
|
||||
// Checked one at a time and against what is already accepted, so one
|
||||
// bad model cannot take a good one with it.
|
||||
@@ -459,6 +469,82 @@ func (s *Service) storedModels(ctx context.Context) ([]catalog.Model, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SetFederated supplies the models federated clusters offer, which join the
|
||||
// catalog after the declared ones and before those added at runtime.
|
||||
//
|
||||
// A function rather than a list, read at every rebuild, because the list
|
||||
// changes whenever a peer's does; Refresh is how federation says it has.
|
||||
func (s *Service) SetFederated(f func() []catalog.Model) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.federated = f
|
||||
}
|
||||
|
||||
// Refresh rebuilds the catalog and puts it in force: after a peer's models
|
||||
// changed. Built as at startup, so a stored model that no longer makes a
|
||||
// valid catalog is skipped and said rather than taking the rest with it.
|
||||
func (s *Service) Refresh(ctx context.Context) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
built, err := s.load(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.apply.ReplaceCatalog(ctx, built)
|
||||
return nil
|
||||
}
|
||||
|
||||
// base is the declared models, then the federated ones whose names nothing
|
||||
// declared or added at runtime already uses.
|
||||
//
|
||||
// A peer's model is named PEER/MODEL, which a local model could also be
|
||||
// called — Hugging Face ids look just like that. The local one wins, for the
|
||||
// reason configuration wins over the database: a request for a model this
|
||||
// cluster runs must not leave it because a peer started offering a name.
|
||||
func (s *Service) base(stored []catalog.Model) []catalog.Model {
|
||||
next := append([]catalog.Model{}, s.declared...)
|
||||
if s.federated == nil {
|
||||
return next
|
||||
}
|
||||
taken := map[string]bool{}
|
||||
for _, list := range [][]catalog.Model{s.declared, stored} {
|
||||
for _, m := range list {
|
||||
taken[m.Name] = true
|
||||
for _, a := range m.Aliases {
|
||||
taken[a] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, f := range s.federated() {
|
||||
if taken[f.Name] {
|
||||
s.log.Warn("a federated model has the name of a model of this cluster's; not offered",
|
||||
"model", f.Name)
|
||||
continue
|
||||
}
|
||||
next = append(next, f)
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
// SetFallbacks replaces the models a model added at runtime falls back to
|
||||
// when it has no capacity: other models of the catalog, a federated peer's
|
||||
// among them.
|
||||
func (s *Service) SetFallbacks(ctx context.Context, name string, fallbacks []string, by string) (catalog.Model, error) {
|
||||
final, err := s.update(ctx, name, by, "change its fallbacks", func(m *catalog.Model) error {
|
||||
for _, f := range fallbacks {
|
||||
if f == m.Name {
|
||||
return fmt.Errorf("model %q cannot fall back to itself", name)
|
||||
}
|
||||
}
|
||||
m.Fallbacks = fallbacks
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
s.log.Info("model fallbacks set", "model", name, "fallbacks", fallbacks, "by", by)
|
||||
}
|
||||
return final, err
|
||||
}
|
||||
|
||||
// SetCheck runs before a model is added, and refuses it by returning an error.
|
||||
// Used to catch weights the engine cannot load, which otherwise are placed and
|
||||
// then fail on every node they are tried on.
|
||||
|
||||
Reference in new issue
Block a user