Federation: another llmgw cluster as a provider, its models as PEER/MODEL
ctl federation add/list/remove/sync keeps peers in the store (migration 0029); their /v1/models are offered here as external models named PEER/MODEL and kept in step, a model still named by a fallback kept withdrawn. Peers are checked at /ready, timed, and taken out of routing while down. Requests to a peer carry x-llmgw-hop and are refused (508) when they come back to a cluster they passed through; residency and no_store keep requests off peers; X-Request-Id and, with --send-tags, attribution tags go along. ctl model fallbacks sets overflow to a peer. Console view, metrics, admin API routes and docs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
40e9fd2ab1
33 files changed
+2944
-23
No files matched your search
@@ -2,7 +2,9 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
@@ -40,6 +42,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fleetmetrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
@@ -168,6 +171,21 @@ func newServerCmd() *cobra.Command {
|
||||
rt.Start(ctx)
|
||||
defer rt.Stop()
|
||||
|
||||
// Which cluster this is, to federated peers: the id in the hop
|
||||
// header that stops a request going round two clusters that
|
||||
// federate each other, and the name in the tags a peer may be
|
||||
// sent. The id is the store's, so every control plane of one
|
||||
// cluster says the same.
|
||||
clusterID, err := federationID(ctx, rt.Store())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
clusterName := cfg.Federation.Name
|
||||
if clusterName == "" {
|
||||
clusterName = host
|
||||
}
|
||||
rt.SetFederationIdentity(clusterID, clusterName)
|
||||
|
||||
// Webhooks for what needs a person, when server.yaml names any.
|
||||
notifier, err := notify.New(ctx, cfg.Notify, log)
|
||||
if err != nil {
|
||||
@@ -325,6 +343,21 @@ func newServerCmd() *cobra.Command {
|
||||
if st := rt.Store(); st != nil {
|
||||
svc := modelsvc.New(cfg.Models, st, admin, log)
|
||||
svc.SetCheck(servableCheck(cfg.WantedEngines()["llamacpp"]))
|
||||
// Other llmgw clusters' models, offered as PEER/MODEL:
|
||||
// read before the catalog is built, so a model that falls
|
||||
// back to one is whole at startup.
|
||||
fed := federation.New(federation.Config{Self: clusterID, Interval: cfg.Federation.Interval},
|
||||
st, svc, admin.Catalog, log)
|
||||
if err := fed.Load(ctx); err != nil {
|
||||
log.Warn("could not read the federation peers; continuing without them", "err", err)
|
||||
}
|
||||
svc.SetFederated(fed.Models)
|
||||
admin.SetPeers(fed)
|
||||
api.SetFederation(fed)
|
||||
if err := rt.RegisterMetrics(fed); err != nil {
|
||||
return err
|
||||
}
|
||||
go fed.Run(ctx)
|
||||
// Rebuild the catalog from configuration plus what was
|
||||
// stored, and put that in force — otherwise models added
|
||||
// before the last restart would be silently absent until
|
||||
@@ -930,3 +963,25 @@ func withLimits(s *http.Server) *http.Server {
|
||||
s.MaxHeaderBytes = 1 << 20
|
||||
return s
|
||||
}
|
||||
|
||||
// federationID is this cluster's id among federated peers: kept in the store,
|
||||
// so every control plane of the cluster and every restart says the same. A
|
||||
// gateway with no store makes one for its lifetime, which is all a router
|
||||
// with no peers of its own needs to refuse a request that comes back to it.
|
||||
func federationID(ctx context.Context, st *storepkg.DB) (string, error) {
|
||||
if st == nil {
|
||||
b := make([]byte, 8)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(b), nil
|
||||
}
|
||||
secret, err := st.ClusterSecret(ctx, "cluster-id")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("this cluster's id for federation: %w", err)
|
||||
}
|
||||
// A prefix is plenty to tell clusters apart, and the id is sent to
|
||||
// peers: nothing is derived from it, but there is no reason to send
|
||||
// more of a secret than needed.
|
||||
return secret[:16], nil
|
||||
}
|
||||
Reference in new issue
Block a user