Federation: another llmgw cluster as a provider, its models as PEER/MODEL

ctl federation add/list/remove/sync keeps peers in the store (migration
0029); their /v1/models are offered here as external models named
PEER/MODEL and kept in step, a model still named by a fallback kept
withdrawn. Peers are checked at /ready, timed, and taken out of routing
while down. Requests to a peer carry x-llmgw-hop and are refused (508) when
they come back to a cluster they passed through; residency and no_store
keep requests off peers; X-Request-Id and, with --send-tags, attribution
tags go along. ctl model fallbacks sets overflow to a peer. Console view,
metrics, admin API routes and docs.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:21:30 +02:00
1 parent d41104a452
commit 40e9fd2ab1
33 files changed
+2944 -23

No files matched your search

+46 -3
View File
@@ -87,6 +87,7 @@ type Controller struct {
disp reconciler.Dispatcher
sink EndpointSink
usage UsageSource
peers Peers
mu sync.Mutex
last string // fingerprint of the last published list
@@ -145,6 +146,27 @@ func (c *Controller) collectUsage() {
}
}
// Peers says which federated clusters' models may be sent requests now.
//
// Federation keeps it: whether each peer answered its last check, and
// whether it still offers the model. A peer model is in the catalog either
// way — a local model may name it as a fallback — and is published only
// while this says yes, which is what takes a peer that is down out of
// routing within a publish.
type Peers interface {
// PeerRoute is the peer's own cluster id and whether requests sent to
// it carry tags, and false when the model is not to be sent anything.
PeerRoute(peer, upstreamModel string) (cluster string, sendTags, ok bool)
}
// SetPeers supplies the federation's view of its peers. Without one, no
// peer model is published.
func (c *Controller) SetPeers(p Peers) {
c.mu.Lock()
c.peers = p
c.mu.Unlock()
}
// LoadSource reports each model's replicas' load, keyed by catalog name.
type LoadSource interface {
ModelLoad() map[string]reconciler.Load
@@ -349,11 +371,25 @@ func (c *Controller) endpoints() []endpoint.Info {
// one each, with the provider's key read from its file every time, so a key
// rotated on disk is used from the next publish on.
func (c *Controller) externalEndpoints() []endpoint.Info {
c.mu.Lock()
peers := c.peers
c.mu.Unlock()
var out []endpoint.Info
for _, m := range c.cat.Load().Models() {
if !m.IsExternal() {
continue
}
var cluster string
var tags bool
if m.IsPeer() {
ok := false
if peers != nil {
cluster, tags, ok = peers.PeerRoute(m.External.Peer, m.External.Model)
}
if !ok {
continue
}
}
key := ""
if f := m.External.APIKeyFile; f != "" {
b, err := secrets.Read(context.Background(), f)
@@ -363,11 +399,18 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
}
key = strings.TrimSpace(string(b))
}
// A peer's endpoint says so in its id, which is what usage records
// as the replica that answered.
id := "external:" + m.Name
if m.IsPeer() {
id = "peer:" + m.Name
}
out = append(out, endpoint.Info{
ID: "external:" + m.Name, URL: strings.TrimRight(m.External.URL, "/"),
ID: id, URL: strings.TrimRight(m.External.URL, "/"),
Models: append([]string{m.Name}, m.Aliases...), APIKey: key,
Engine: endpoint.EngineExternal, Pool: "external", Weight: 1,
UpstreamModel: m.External.Model,
Peer: m.External.Peer, PeerCluster: cluster, PeerTags: tags,
})
}
return out
@@ -377,8 +420,8 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
func fingerprint(infos []endpoint.Info) string {
var b strings.Builder
for _, i := range infos {
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d|%s|%x;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority,
i.UpstreamModel, sha256.Sum256([]byte(i.APIKey)))
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d|%s|%s|%s|%t|%x;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight,
i.Priority, i.UpstreamModel, i.Peer, i.PeerCluster, i.PeerTags, sha256.Sum256([]byte(i.APIKey)))
}
return b.String()
}