Federation: another llmgw cluster as a provider, its models as PEER/MODEL

ctl federation add/list/remove/sync keeps peers in the store (migration
0029); their /v1/models are offered here as external models named
PEER/MODEL and kept in step, a model still named by a fallback kept
withdrawn. Peers are checked at /ready, timed, and taken out of routing
while down. Requests to a peer carry x-llmgw-hop and are refused (508) when
they come back to a cluster they passed through; residency and no_store
keep requests off peers; X-Request-Id and, with --send-tags, attribution
tags go along. ctl model fallbacks sets overflow to a peer. Console view,
metrics, admin API routes and docs.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:21:30 +02:00
1 parent d41104a452
commit 40e9fd2ab1
33 files changed
+2944 -23

No files matched your search

+90 -4
View File
@@ -51,6 +51,9 @@ type Service struct {
// SetCheck.
check func(context.Context, catalog.Model) error
// federated are the models federated clusters offer; see SetFederated.
federated func() []catalog.Model
// mu serialises rebuilds. They are rare — a person adding a model — and two
// at once could apply catalogs in either order, leaving the one in force
// disagreeing with the database that outlives it.
@@ -128,7 +131,7 @@ func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error {
if err != nil {
return err
}
next := append([]catalog.Model{}, s.declared...)
next := s.base(stored)
for _, e := range stored {
if e.Name == m.Name {
continue // this call replaces it
@@ -174,7 +177,7 @@ func (s *Service) Remove(ctx context.Context, name string) error {
if err != nil {
return err
}
next := append([]catalog.Model{}, s.declared...)
next := s.base(stored)
found := false
for _, e := range stored {
if e.Name == name {
@@ -371,7 +374,7 @@ func (s *Service) update(ctx context.Context, name, by, what string, change func
if err != nil {
return catalog.Model{}, err
}
next := append([]catalog.Model{}, s.declared...)
next := s.base(stored)
found := false
for _, e := range stored {
if e.Name == name {
@@ -416,12 +419,19 @@ func (s *Service) Load(ctx context.Context) (*catalog.Catalog, error) {
s.mu.Lock()
defer s.mu.Unlock()
return s.load(ctx)
}
// load is Load, with the lock held.
func (s *Service) load(ctx context.Context) (*catalog.Catalog, error) {
stored, err := s.storedModels(ctx)
if err != nil {
return nil, err
}
next := append([]catalog.Model{}, s.declared...)
// The federated models before the stored ones, so a stored model that
// falls back to a peer's is checked against a catalog that has it.
next := s.base(stored)
for _, m := range stored {
// Checked one at a time and against what is already accepted, so one
// bad model cannot take a good one with it.
@@ -459,6 +469,82 @@ func (s *Service) storedModels(ctx context.Context) ([]catalog.Model, error) {
return out, nil
}
// SetFederated supplies the models federated clusters offer, which join the
// catalog after the declared ones and before those added at runtime.
//
// A function rather than a list, read at every rebuild, because the list
// changes whenever a peer's does; Refresh is how federation says it has.
func (s *Service) SetFederated(f func() []catalog.Model) {
s.mu.Lock()
defer s.mu.Unlock()
s.federated = f
}
// Refresh rebuilds the catalog and puts it in force: after a peer's models
// changed. Built as at startup, so a stored model that no longer makes a
// valid catalog is skipped and said rather than taking the rest with it.
func (s *Service) Refresh(ctx context.Context) error {
s.mu.Lock()
defer s.mu.Unlock()
built, err := s.load(ctx)
if err != nil {
return err
}
s.apply.ReplaceCatalog(ctx, built)
return nil
}
// base is the declared models, then the federated ones whose names nothing
// declared or added at runtime already uses.
//
// A peer's model is named PEER/MODEL, which a local model could also be
// called — Hugging Face ids look just like that. The local one wins, for the
// reason configuration wins over the database: a request for a model this
// cluster runs must not leave it because a peer started offering a name.
func (s *Service) base(stored []catalog.Model) []catalog.Model {
next := append([]catalog.Model{}, s.declared...)
if s.federated == nil {
return next
}
taken := map[string]bool{}
for _, list := range [][]catalog.Model{s.declared, stored} {
for _, m := range list {
taken[m.Name] = true
for _, a := range m.Aliases {
taken[a] = true
}
}
}
for _, f := range s.federated() {
if taken[f.Name] {
s.log.Warn("a federated model has the name of a model of this cluster's; not offered",
"model", f.Name)
continue
}
next = append(next, f)
}
return next
}
// SetFallbacks replaces the models a model added at runtime falls back to
// when it has no capacity: other models of the catalog, a federated peer's
// among them.
func (s *Service) SetFallbacks(ctx context.Context, name string, fallbacks []string, by string) (catalog.Model, error) {
final, err := s.update(ctx, name, by, "change its fallbacks", func(m *catalog.Model) error {
for _, f := range fallbacks {
if f == m.Name {
return fmt.Errorf("model %q cannot fall back to itself", name)
}
}
m.Fallbacks = fallbacks
return nil
})
if err == nil {
s.log.Info("model fallbacks set", "model", name, "fallbacks", fallbacks, "by", by)
}
return final, err
}
// SetCheck runs before a model is added, and refuses it by returning an error.
// Used to catch weights the engine cannot load, which otherwise are placed and
// then fail on every node they are tried on.