Federation: another llmgw cluster as a provider, its models as PEER/MODEL

ctl federation add/list/remove/sync keeps peers in the store (migration
0029); their /v1/models are offered here as external models named
PEER/MODEL and kept in step, a model still named by a fallback kept
withdrawn. Peers are checked at /ready, timed, and taken out of routing
while down. Requests to a peer carry x-llmgw-hop and are refused (508) when
they come back to a cluster they passed through; residency and no_store
keep requests off peers; X-Request-Id and, with --send-tags, attribution
tags go along. ctl model fallbacks sets overflow to a peer. Console view,
metrics, admin API routes and docs.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:21:30 +02:00
1 parent d41104a452
commit 40e9fd2ab1
33 files changed
+2944 -23

No files matched your search

+55
View File
@@ -2,7 +2,9 @@ package cli
import (
"context"
"crypto/rand"
"crypto/tls"
"encoding/hex"
"errors"
"fmt"
"log/slog"
@@ -40,6 +42,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fleetmetrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
@@ -168,6 +171,21 @@ func newServerCmd() *cobra.Command {
rt.Start(ctx)
defer rt.Stop()
// Which cluster this is, to federated peers: the id in the hop
// header that stops a request going round two clusters that
// federate each other, and the name in the tags a peer may be
// sent. The id is the store's, so every control plane of one
// cluster says the same.
clusterID, err := federationID(ctx, rt.Store())
if err != nil {
return err
}
clusterName := cfg.Federation.Name
if clusterName == "" {
clusterName = host
}
rt.SetFederationIdentity(clusterID, clusterName)
// Webhooks for what needs a person, when server.yaml names any.
notifier, err := notify.New(ctx, cfg.Notify, log)
if err != nil {
@@ -325,6 +343,21 @@ func newServerCmd() *cobra.Command {
if st := rt.Store(); st != nil {
svc := modelsvc.New(cfg.Models, st, admin, log)
svc.SetCheck(servableCheck(cfg.WantedEngines()["llamacpp"]))
// Other llmgw clusters' models, offered as PEER/MODEL:
// read before the catalog is built, so a model that falls
// back to one is whole at startup.
fed := federation.New(federation.Config{Self: clusterID, Interval: cfg.Federation.Interval},
st, svc, admin.Catalog, log)
if err := fed.Load(ctx); err != nil {
log.Warn("could not read the federation peers; continuing without them", "err", err)
}
svc.SetFederated(fed.Models)
admin.SetPeers(fed)
api.SetFederation(fed)
if err := rt.RegisterMetrics(fed); err != nil {
return err
}
go fed.Run(ctx)
// Rebuild the catalog from configuration plus what was
// stored, and put that in force — otherwise models added
// before the last restart would be silently absent until
@@ -930,3 +963,25 @@ func withLimits(s *http.Server) *http.Server {
s.MaxHeaderBytes = 1 << 20
return s
}
// federationID is this cluster's id among federated peers: kept in the store,
// so every control plane of the cluster and every restart says the same. A
// gateway with no store makes one for its lifetime, which is all a router
// with no peers of its own needs to refuse a request that comes back to it.
func federationID(ctx context.Context, st *storepkg.DB) (string, error) {
if st == nil {
b := make([]byte, 8)
if _, err := rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
secret, err := st.ClusterSecret(ctx, "cluster-id")
if err != nil {
return "", fmt.Errorf("this cluster's id for federation: %w", err)
}
// A prefix is plenty to tell clusters apart, and the id is sent to
// peers: nothing is derived from it, but there is no reason to send
// more of a secret than needed.
return secret[:16], nil
}