Federation: another llmgw cluster as a provider, its models as PEER/MODEL

ctl federation add/list/remove/sync keeps peers in the store (migration
0029); their /v1/models are offered here as external models named
PEER/MODEL and kept in step, a model still named by a fallback kept
withdrawn. Peers are checked at /ready, timed, and taken out of routing
while down. Requests to a peer carry x-llmgw-hop and are refused (508) when
they come back to a cluster they passed through; residency and no_store
keep requests off peers; X-Request-Id and, with --send-tags, attribution
tags go along. ctl model fallbacks sets overflow to a peer. Console view,
metrics, admin API routes and docs.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:21:30 +02:00
1 parent d41104a452
commit 40e9fd2ab1
33 files changed
+2944 -23

No files matched your search

+8
View File
@@ -32,6 +32,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
@@ -58,6 +59,9 @@ type API struct {
history History
// audit reads the audit log.
audit AuditReader
// federation is the other llmgw clusters whose models are offered here;
// nil without a store.
federation *federation.Service
// guard is the guardrail policies' library and attachments.
guard *GuardAdmin
// revoke records a removed node, so its certificate is refused.
@@ -172,6 +176,7 @@ func (a *API) Register(mux Mux) {
mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel)
a.registerCanary(mux)
a.registerEval(mux)
a.registerFederation(mux)
mux.HandleFunc("POST /api/v1/models/{name}/origin", a.setOrigin)
mux.HandleFunc("GET /api/v1/plan", a.planModel)
mux.HandleFunc("POST /api/v1/metrics/range", a.metricsBatch)
@@ -1810,6 +1815,9 @@ func externalLabel(m catalog.Model) string {
if !m.IsExternal() {
return ""
}
if m.IsPeer() {
return "peer " + m.External.Peer + " · " + m.External.Model
}
host := m.External.URL
if u, err := url.Parse(host); err == nil && u.Host != "" {
host = u.Host
+123
View File
@@ -0,0 +1,123 @@
package adminapi
import (
"context"
"encoding/json"
"io"
"net/http"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
)
// Federation: other llmgw clusters whose models this one offers, as
// PEER/MODEL, and the fallbacks that send overflow to them.
// SetFederation gives the API the peers to list and change.
func (a *API) SetFederation(f *federation.Service) { a.federation = f }
// FederationView is this cluster's id and its peers.
type FederationView struct {
// Cluster is this cluster's own id, which its peers see in the hop
// header.
Cluster string `json:"cluster"`
Peers []federation.Status `json:"peers"`
}
// FallbacksRequest replaces a model's fallbacks; empty takes them all away.
type FallbacksRequest struct {
Fallbacks []string `json:"fallbacks"`
}
// FallbacksView is a model's fallbacks, as they are now.
type FallbacksView struct {
Model string `json:"model"`
Fallbacks []string `json:"fallbacks"`
}
type fallbackSetter interface {
SetFallbacks(ctx context.Context, name string, fallbacks []string, by string) (catalog.Model, error)
}
func (a *API) registerFederation(mux Mux) {
need := func(w http.ResponseWriter) bool {
if a.federation == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store to keep federation peers in")
return false
}
return true
}
mux.HandleFunc("GET /api/v1/federation", func(w http.ResponseWriter, _ *http.Request) {
if !need(w) {
return
}
writeJSON(w, http.StatusOK, FederationView{Cluster: a.federation.Self(), Peers: a.federation.List()})
})
mux.HandleFunc("POST /api/v1/federation", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
var req federation.AddRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<14)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
st, err := a.federation.Add(r.Context(), req, actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusCreated, st)
})
mux.HandleFunc("DELETE /api/v1/federation/{peer}", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
name := r.PathValue("peer")
if err := a.federation.Remove(r.Context(), name); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, Removed{Removed: name})
})
mux.HandleFunc("POST /api/v1/federation/{peer}/sync", func(w http.ResponseWriter, r *http.Request) {
if !need(w) {
return
}
st, err := a.federation.Sync(r.Context(), r.PathValue("peer"))
if err != nil {
writeError(w, http.StatusNotFound, err.Error())
return
}
writeJSON(w, http.StatusOK, st)
})
mux.HandleFunc("PUT /api/v1/models/{name}/fallbacks", func(w http.ResponseWriter, r *http.Request) {
setter, ok := a.models.(fallbackSetter)
if !ok {
writeError(w, http.StatusNotImplemented, "this control plane has no store to keep the setting in")
return
}
var req FallbacksRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<14)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
var fbs []string
for _, f := range req.Fallbacks {
if f = strings.TrimSpace(f); f != "" {
fbs = append(fbs, f)
}
}
m, err := setter.SetFallbacks(r.Context(), r.PathValue("name"), fbs, actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
out := m.Fallbacks
if out == nil {
out = []string{}
}
writeJSON(w, http.StatusOK, FallbacksView{Model: m.Name, Fallbacks: out})
})
}
+24 -7
View File
@@ -8,6 +8,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
@@ -56,13 +57,14 @@ type Param struct {
// PathParams describes the parameters in Routes' paths, by the segment
// before them and their name.
var PathParams = map[string]string{
"models/{name}": "The model's name in the catalog.",
"nodes/{id}": "The node's ID, as `llmgw ctl nodes` lists it.",
"jobs/{id}": "The job's ID, as starting it answered.",
"runs/{id}": "The quality run's ID, as starting it answered.",
"requests/{id}": "The request's ID: the x-request-id a client was answered with.",
"fabrics/{id}": "The physical fabric's ID, as GET /api/v1/fabrics lists it.",
"quotas/{scope}": "The scope: org:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT, or their IDs after the colon.",
"models/{name}": "The model's name in the catalog.",
"nodes/{id}": "The node's ID, as `llmgw ctl nodes` lists it.",
"jobs/{id}": "The job's ID, as starting it answered.",
"runs/{id}": "The quality run's ID, as starting it answered.",
"requests/{id}": "The request's ID: the x-request-id a client was answered with.",
"fabrics/{id}": "The physical fabric's ID, as GET /api/v1/fabrics lists it.",
"quotas/{scope}": "The scope: org:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT, or their IDs after the colon.",
"federation/{peer}": "The federation peer's name, as `llmgw ctl federation list` lists it.",
}
// Routes is every endpoint Register mounts.
@@ -210,6 +212,21 @@ var Routes = []Route{
{Method: "DELETE", Path: "/api/v1/models/{name}/eval-gate", Summary: "Stop holding a model's updates to a suite",
Response: EvalGateView{}},
// Federation: other llmgw clusters as providers.
{Method: "GET", Path: "/api/v1/federation", Summary: "This cluster's id, and its federation peers with their health and models",
Response: FederationView{}},
{Method: "POST", Path: "/api/v1/federation", Summary: "Add another llmgw cluster as a peer, or change one",
Description: "The peer is checked first: its /ready, timed, and its /v1/models with the key. Its models join the catalog as PEER/MODEL.",
Request: federation.AddRequest{}, Response: federation.Status{}, Status: http.StatusCreated},
{Method: "DELETE", Path: "/api/v1/federation/{peer}", Summary: "Remove a federation peer and its models",
Description: "Refused while a model here falls back to one of the peer's.",
Response: Removed{}},
{Method: "POST", Path: "/api/v1/federation/{peer}/sync", Summary: "Check a peer now, rather than at the next interval",
Response: federation.Status{}},
{Method: "PUT", Path: "/api/v1/models/{name}/fallbacks", Summary: "Replace the models a model falls back to when it has no capacity",
Description: "A model added at runtime only. A federation peer's model, PEER/MODEL, sends the overflow to the other cluster.",
Request: FallbacksRequest{}, Response: FallbacksView{}},
// Tenancy: quotas and guardrails.
{Method: "GET", Path: "/api/v1/quotas", Summary: "Every capacity quota, with what its scope holds now",
Response: []QuotaView{}},
+8
View File
@@ -1017,8 +1017,16 @@ type External struct {
// rather than the key itself, so the key is in neither the catalog, the
// store nor the audit log; empty for a provider that asks for none.
APIKeyFile string `json:"api_key_file,omitempty" koanf:"api_key_file"`
// Peer names the federated llmgw cluster this model is offered by, for
// a model that federation added (`llmgw ctl federation add`) rather
// than one written by hand. Such a model is named PEER/MODEL, comes and
// goes with the peer's own list, and is sent only while the peer is up.
Peer string `json:"peer,omitempty" koanf:"peer"`
}
// IsPeer reports whether a model is offered by a federated cluster.
func (m Model) IsPeer() bool { return m.External != nil && m.External.Peer != "" }
func (e *External) validate(model string, profiles int) error {
if profiles > 0 {
return fmt.Errorf("model %q: an external model is served elsewhere and has no profiles", model)
+46 -3
View File
@@ -87,6 +87,7 @@ type Controller struct {
disp reconciler.Dispatcher
sink EndpointSink
usage UsageSource
peers Peers
mu sync.Mutex
last string // fingerprint of the last published list
@@ -145,6 +146,27 @@ func (c *Controller) collectUsage() {
}
}
// Peers says which federated clusters' models may be sent requests now.
//
// Federation keeps it: whether each peer answered its last check, and
// whether it still offers the model. A peer model is in the catalog either
// way — a local model may name it as a fallback — and is published only
// while this says yes, which is what takes a peer that is down out of
// routing within a publish.
type Peers interface {
// PeerRoute is the peer's own cluster id and whether requests sent to
// it carry tags, and false when the model is not to be sent anything.
PeerRoute(peer, upstreamModel string) (cluster string, sendTags, ok bool)
}
// SetPeers supplies the federation's view of its peers. Without one, no
// peer model is published.
func (c *Controller) SetPeers(p Peers) {
c.mu.Lock()
c.peers = p
c.mu.Unlock()
}
// LoadSource reports each model's replicas' load, keyed by catalog name.
type LoadSource interface {
ModelLoad() map[string]reconciler.Load
@@ -349,11 +371,25 @@ func (c *Controller) endpoints() []endpoint.Info {
// one each, with the provider's key read from its file every time, so a key
// rotated on disk is used from the next publish on.
func (c *Controller) externalEndpoints() []endpoint.Info {
c.mu.Lock()
peers := c.peers
c.mu.Unlock()
var out []endpoint.Info
for _, m := range c.cat.Load().Models() {
if !m.IsExternal() {
continue
}
var cluster string
var tags bool
if m.IsPeer() {
ok := false
if peers != nil {
cluster, tags, ok = peers.PeerRoute(m.External.Peer, m.External.Model)
}
if !ok {
continue
}
}
key := ""
if f := m.External.APIKeyFile; f != "" {
b, err := secrets.Read(context.Background(), f)
@@ -363,11 +399,18 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
}
key = strings.TrimSpace(string(b))
}
// A peer's endpoint says so in its id, which is what usage records
// as the replica that answered.
id := "external:" + m.Name
if m.IsPeer() {
id = "peer:" + m.Name
}
out = append(out, endpoint.Info{
ID: "external:" + m.Name, URL: strings.TrimRight(m.External.URL, "/"),
ID: id, URL: strings.TrimRight(m.External.URL, "/"),
Models: append([]string{m.Name}, m.Aliases...), APIKey: key,
Engine: endpoint.EngineExternal, Pool: "external", Weight: 1,
UpstreamModel: m.External.Model,
Peer: m.External.Peer, PeerCluster: cluster, PeerTags: tags,
})
}
return out
@@ -377,8 +420,8 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
func fingerprint(infos []endpoint.Info) string {
var b strings.Builder
for _, i := range infos {
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d|%s|%x;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority,
i.UpstreamModel, sha256.Sum256([]byte(i.APIKey)))
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d|%s|%s|%s|%t|%x;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight,
i.Priority, i.UpstreamModel, i.Peer, i.PeerCluster, i.PeerTags, sha256.Sum256([]byte(i.APIKey)))
}
return b.String()
}
+661
View File
@@ -0,0 +1,661 @@
// Package federation makes other llmgw clusters providers of this one.
//
// A peer is another llmgw gateway, reached with a key it issued to this
// cluster. Its models are offered here as external models named PEER/MODEL,
// kept in step with its own /v1/models, and sent requests only while its
// /ready answers. A local model names one as a fallback to send overflow to
// the other site.
//
// Always PEER/MODEL, never the peer's own name for a model, even when nothing
// here uses that name. A name that left the cluster or stayed depending on
// what the catalog happened to hold would move traffic between sites when
// somebody added an unrelated model; the prefix also says, on every usage row
// and every x-llmgw-served-model header, which site answered.
//
// Everything that keeps a request inside the cluster already applies: a peer
// model is an external model, so a project whose requests stay in the
// cluster, or a policy that keeps nothing, is never sent to one.
package federation
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"regexp"
"slices"
"strings"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/secrets"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// ClusterHeader is the response header in which a gateway says its own
// cluster id, on /v1/models. Federation reads a peer's from it, to know the
// peer in a request's hop header.
const ClusterHeader = "x-llmgw-cluster"
// Store is the persistence federation needs.
type Store interface {
PutFederationPeer(ctx context.Context, p store.FederationPeer) error
SetFederationPeerModels(ctx context.Context, name, cluster string, models json.RawMessage, at time.Time) error
DeleteFederationPeer(ctx context.Context, name string) error
FederationPeers(ctx context.Context) ([]store.FederationPeer, error)
}
// Catalog is what puts a changed peer list in force: the model service.
type Catalog interface {
Refresh(ctx context.Context) error
}
// Config tunes federation.
type Config struct {
// Self is this cluster's id. A peer that says the same is this cluster,
// reached by another address, and is refused.
Self string
// Interval is how often each peer is checked; zero is 10s.
Interval time.Duration
// Timeout bounds one check of one peer; zero is 5s.
Timeout time.Duration
// Client makes the checks; nil is a plain one.
Client *http.Client
}
// PeerModel is what a peer says about one model it offers.
type PeerModel struct {
// ID is the peer's name for it, and what it is asked for.
ID string `json:"id"`
Description string `json:"description,omitempty"`
ContextLength int `json:"context_length,omitempty"`
MaxOutput int `json:"max_output,omitempty"`
Capabilities catalog.Capabilities `json:"capabilities,omitempty"`
// Withdrawn is a model the peer no longer offers, kept in the catalog
// because a model here still falls back to it: removing it would make
// that model invalid. Nothing is sent to it, and it goes when nothing
// names it any more.
Withdrawn bool `json:"withdrawn,omitempty"`
}
// Service keeps the peers, their health and their models.
type Service struct {
cfg Config
store Store
catalog Catalog
// current is the catalog in force, read to tell which of a peer's
// models a local model names.
current func() *catalog.Catalog
log *slog.Logger
// op serialises what changes the peer set or a peer's models, so two
// rebuilds cannot land in the wrong order.
op sync.Mutex
mu sync.Mutex
peers map[string]*peer
}
type peer struct {
rec store.FederationPeer
models []PeerModel
health Health
}
// Health is how a peer answered its last check.
type Health struct {
Up bool `json:"up"`
// Checked is when; zero before the first check.
Checked time.Time `json:"checked,omitempty"`
// Latency is how long the peer took to answer /ready: the network to
// the other site and the peer's front, not a model's speed.
Latency time.Duration `json:"latency_ns,omitempty"`
// Error is why the peer is down, empty while it is up.
Error string `json:"error,omitempty"`
}
// New builds the service. Load reads the peers kept in the store.
func New(cfg Config, st Store, cat Catalog, current func() *catalog.Catalog, log *slog.Logger) *Service {
if cfg.Interval <= 0 {
cfg.Interval = 10 * time.Second
}
if cfg.Timeout <= 0 {
cfg.Timeout = 5 * time.Second
}
if cfg.Client == nil {
cfg.Client = &http.Client{}
}
if log == nil {
log = slog.Default()
}
return &Service{cfg: cfg, store: st, catalog: cat, current: current, log: log, peers: map[string]*peer{}}
}
// Self is this cluster's id.
func (s *Service) Self() string { return s.cfg.Self }
// Load reads the peers and the models they last offered from the store, so
// the catalog built at startup has them before any peer is asked again.
// Every peer reads as down until its first check.
func (s *Service) Load(ctx context.Context) error {
recs, err := s.store.FederationPeers(ctx)
if err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
for _, r := range recs {
var models []PeerModel
if err := json.Unmarshal(r.Models, &models); err != nil {
s.log.Warn("a federation peer's models could not be read; they come back at its next check",
"peer", r.Name, "err", err)
}
s.peers[r.Name] = &peer{rec: r, models: models, health: Health{Error: "not checked yet"}}
}
return nil
}
// Models are the peers' models as catalog entries, withdrawn ones included:
// what the model service adds to the catalog.
func (s *Service) Models() []catalog.Model {
s.mu.Lock()
defer s.mu.Unlock()
var out []catalog.Model
for _, name := range s.namesLocked() {
p := s.peers[name]
for _, m := range p.models {
out = append(out, catalogModel(p.rec, m))
}
}
return out
}
func (s *Service) namesLocked() []string {
names := make([]string, 0, len(s.peers))
for n := range s.peers {
names = append(names, n)
}
slices.Sort(names)
return names
}
// catalogModel is a peer's model as an entry of this catalog.
func catalogModel(rec store.FederationPeer, m PeerModel) catalog.Model {
desc := "on " + rec.Name
if m.Description != "" {
desc += ": " + m.Description
}
return catalog.Model{
Name: rec.Name + "/" + m.ID, Source: "peer:" + rec.Name, Description: desc,
ContextLength: m.ContextLength, MaxOutput: m.MaxOutput, Capabilities: m.Capabilities,
External: &catalog.External{URL: rec.URL, Model: m.ID, APIKeyFile: rec.KeyFile, Peer: rec.Name},
}
}
// PeerRoute says whether a peer's model may be sent requests now: the peer
// answered its last check and still offers it. The controller publishes the
// model's endpoint only while it does.
func (s *Service) PeerRoute(name, model string) (cluster string, sendTags, ok bool) {
s.mu.Lock()
defer s.mu.Unlock()
p, found := s.peers[name]
if !found || !p.health.Up {
return "", false, false
}
for _, m := range p.models {
if m.ID == model {
return p.rec.Cluster, p.rec.SendTags, !m.Withdrawn
}
}
return "", false, false
}
// peerName is what a peer may be called: the prefix of its models' names,
// and a tag value, so letters, digits and a few separators, and no slash.
var peerName = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]{0,31}$`)
// AddRequest adds a peer.
type AddRequest struct {
// Name is what this cluster calls the peer, and the prefix of its
// models: NAME/MODEL.
Name string `json:"name"`
// URL is the peer gateway's client address, without /v1.
URL string `json:"url"`
// KeyFile is a file on the control plane holding a key the peer issued
// to this cluster (or env:NAME, vault:PATH#FIELD).
KeyFile string `json:"key_file"`
// SendTags makes requests sent to the peer carry x-llmgw-tags naming
// this cluster and the sending project. Off unless asked: the peer
// learns who in this cluster is asking.
SendTags bool `json:"send_tags,omitempty"`
}
// Add checks a peer and keeps it, its models joining the catalog.
//
// Checked before anything is kept: a mistyped address or a key the peer
// refuses is said now, rather than found as a peer that is always down.
func (s *Service) Add(ctx context.Context, req AddRequest, by string) (Status, error) {
req.Name = strings.TrimSpace(req.Name)
req.URL = strings.TrimRight(strings.TrimSpace(req.URL), "/")
req.URL = strings.TrimSuffix(req.URL, "/v1")
if !peerName.MatchString(req.Name) {
return Status{}, fmt.Errorf("peer name %q: 1 to 32 lower-case letters, digits, '_', '.' or '-', starting with a letter or digit", req.Name)
}
u, err := url.Parse(req.URL)
if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" {
return Status{}, fmt.Errorf("peer address %q is not an http(s) address", req.URL)
}
if req.KeyFile == "" {
return Status{}, errors.New("give the file holding the key the peer issued to this cluster (--key-file)")
}
if cur := s.current(); cur != nil {
for _, m := range cur.Models() {
if !m.IsPeer() && strings.HasPrefix(m.Name, req.Name+"/") {
return Status{}, fmt.Errorf("model %q of this cluster already starts with %s/; call the peer something else",
m.Name, req.Name)
}
}
}
s.op.Lock()
defer s.op.Unlock()
rec := store.FederationPeer{Name: req.Name, URL: req.URL, KeyFile: req.KeyFile, SendTags: req.SendTags,
CreatedAt: time.Now(), CreatedBy: by}
s.mu.Lock()
old, existed := s.peers[req.Name]
if existed {
rec.CreatedAt, rec.CreatedBy = old.rec.CreatedAt, old.rec.CreatedBy
}
s.mu.Unlock()
res := s.check(ctx, rec)
if res.err != nil {
return Status{}, fmt.Errorf("peer %s: %w", req.Name, res.err)
}
if res.cluster != "" && res.cluster == s.cfg.Self {
return Status{}, fmt.Errorf("%s is this cluster: its gateway answers with this cluster's own id", req.URL)
}
rec.Cluster, rec.SyncedAt = res.cluster, time.Now()
models := res.models
if existed {
models = s.keepReferenced(old, res.models)
}
b, _ := json.Marshal(models)
rec.Models = b
if err := s.store.PutFederationPeer(ctx, rec); err != nil {
return Status{}, err
}
s.mu.Lock()
s.peers[req.Name] = &peer{rec: rec, models: models, health: res.health}
s.mu.Unlock()
if err := s.catalog.Refresh(ctx); err != nil {
return Status{}, fmt.Errorf("the peer is kept, but the catalog could not be rebuilt: %w", err)
}
s.log.Info("federation peer added", "peer", req.Name, "url", req.URL, "models", len(models), "by", by)
st, _ := s.Status(req.Name)
return st, nil
}
// Remove takes a peer and its models away.
//
// Refused while a model here falls back to one of them, or has one as its
// canary, for the reason removing a model is: a fallback pointing at
// nothing is found during the outage it was meant to cover.
func (s *Service) Remove(ctx context.Context, name string) error {
s.op.Lock()
defer s.op.Unlock()
s.mu.Lock()
p, ok := s.peers[name]
s.mu.Unlock()
if !ok {
return fmt.Errorf("no federation peer called %q: `llmgw ctl federation list` lists them", name)
}
var users []string
if cur := s.current(); cur != nil {
for _, m := range cur.Models() {
if m.IsPeer() {
continue
}
for _, ref := range references(m) {
if strings.HasPrefix(ref, name+"/") {
users = append(users, m.Name+" → "+ref)
}
}
}
}
if len(users) > 0 {
return fmt.Errorf("models here still use %s's: %s; change their fallbacks first (`llmgw ctl model fallbacks`)",
name, strings.Join(users, ", "))
}
if err := s.store.DeleteFederationPeer(ctx, name); err != nil && !errors.Is(err, store.ErrNotFound) {
return err
}
s.mu.Lock()
delete(s.peers, name)
s.mu.Unlock()
if err := s.catalog.Refresh(ctx); err != nil {
return fmt.Errorf("the peer is removed, but the catalog could not be rebuilt: %w", err)
}
s.log.Info("federation peer removed", "peer", name, "models", len(p.models))
return nil
}
// references are the other models a model names: its fallbacks and its
// canary.
func references(m catalog.Model) []string {
out := slices.Clone(m.Fallbacks)
if m.Canary != nil {
out = append(out, m.Canary.Model)
}
return out
}
// keepReferenced is what a peer offers now, plus those it offered before and
// no longer does that a model here still names, kept withdrawn.
func (s *Service) keepReferenced(old *peer, offered []PeerModel) []PeerModel {
out := slices.Clone(offered)
named := map[string]bool{}
if cur := s.current(); cur != nil {
for _, m := range cur.Models() {
if m.IsPeer() {
continue
}
for _, ref := range references(m) {
named[ref] = true
}
}
}
for _, m := range old.models {
if slices.ContainsFunc(offered, func(o PeerModel) bool { return o.ID == m.ID }) {
continue
}
if named[old.rec.Name+"/"+m.ID] {
m.Withdrawn = true
out = append(out, m)
}
}
return out
}
// Sync checks one peer now, rather than at the next interval.
func (s *Service) Sync(ctx context.Context, name string) (Status, error) {
s.mu.Lock()
p, ok := s.peers[name]
var rec store.FederationPeer
if ok {
rec = p.rec
}
s.mu.Unlock()
if !ok {
return Status{}, fmt.Errorf("no federation peer called %q: `llmgw ctl federation list` lists them", name)
}
res := s.check(ctx, rec)
s.apply(ctx, map[string]checkResult{name: res})
st, _ := s.Status(name)
return st, nil
}
// SyncAll checks every peer at once, and puts what changed in force.
func (s *Service) SyncAll(ctx context.Context) {
s.mu.Lock()
recs := make([]store.FederationPeer, 0, len(s.peers))
for _, p := range s.peers {
recs = append(recs, p.rec)
}
s.mu.Unlock()
results := make(map[string]checkResult, len(recs))
var mu sync.Mutex
var wg sync.WaitGroup
for _, rec := range recs {
wg.Go(func() {
res := s.check(ctx, rec)
mu.Lock()
results[rec.Name] = res
mu.Unlock()
})
}
wg.Wait()
s.apply(ctx, results)
}
// apply records checks' results: each peer's health always, and its models
// when it said them, rebuilding the catalog once if any changed.
func (s *Service) apply(ctx context.Context, results map[string]checkResult) {
s.op.Lock()
defer s.op.Unlock()
type change struct {
name, cluster string
models []PeerModel
}
var changes []change
s.mu.Lock()
for name, res := range results {
p, ok := s.peers[name]
if !ok {
continue // removed while it was being checked
}
if p.health.Up != res.health.Up {
if res.health.Up {
s.log.Info("federation peer is up", "peer", name, "latency", res.health.Latency)
} else {
s.log.Warn("federation peer is down; its models are not sent requests", "peer", name, "err", res.health.Error)
}
}
p.health = res.health
if res.err != nil {
continue // its list is kept as it was: down is not empty
}
next := s.keepReferenced(p, res.models)
if res.cluster != p.rec.Cluster || !slices.Equal(next, p.models) {
p.models, p.rec.Cluster, p.rec.SyncedAt = next, res.cluster, time.Now()
changes = append(changes, change{name, res.cluster, next})
}
}
s.mu.Unlock()
if len(changes) == 0 {
return
}
for _, c := range changes {
b, _ := json.Marshal(c.models)
if err := s.store.SetFederationPeerModels(ctx, c.name, c.cluster, b, time.Now()); err != nil {
s.log.Warn("could not keep a federation peer's models", "peer", c.name, "err", err)
}
s.log.Info("federation peer's models changed", "peer", c.name, "models", len(c.models))
}
if err := s.catalog.Refresh(ctx); err != nil {
s.log.Warn("the catalog could not be rebuilt after a peer's models changed", "err", err)
}
}
// checkResult is one check of one peer.
type checkResult struct {
health Health
cluster string
models []PeerModel
// err is set when the models could not be read; health says the peer
// is down then too.
err error
}
// check asks a peer whether it is ready, timed, and what it offers this
// cluster's key.
func (s *Service) check(ctx context.Context, rec store.FederationPeer) checkResult {
ctx, cancel := context.WithTimeout(ctx, s.cfg.Timeout)
defer cancel()
res := checkResult{health: Health{Checked: time.Now()}}
fail := func(err error) checkResult {
res.health.Up, res.health.Error, res.err = false, err.Error(), err
return res
}
// /ready first, and timed: it is open, cheap, and says whether the peer
// has anything of its own to serve with.
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rec.URL+"/ready", nil)
if err != nil {
return fail(err)
}
start := time.Now()
resp, err := s.cfg.Client.Do(req)
if err != nil {
return fail(fmt.Errorf("the peer did not answer: %w", err))
}
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<12))
resp.Body.Close()
res.health.Latency = time.Since(start)
if resp.StatusCode != http.StatusOK {
return fail(fmt.Errorf("the peer is not ready (%s): it has nothing of its own to serve with", resp.Status))
}
key := ""
if rec.KeyFile != "" {
b, err := secrets.Read(ctx, rec.KeyFile)
if err != nil {
return fail(fmt.Errorf("the key for the peer could not be read: %w", err))
}
key = strings.TrimSpace(string(b))
}
req, err = http.NewRequestWithContext(ctx, http.MethodGet, rec.URL+"/v1/models", nil)
if err != nil {
return fail(err)
}
if key != "" {
req.Header.Set("Authorization", "Bearer "+key)
}
resp, err = s.cfg.Client.Do(req)
if err != nil {
return fail(fmt.Errorf("the peer's models could not be read: %w", err))
}
defer resp.Body.Close()
switch {
case resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden:
return fail(fmt.Errorf("the peer refused this cluster's key (%s): check the key file", resp.Status))
case resp.StatusCode != http.StatusOK:
return fail(fmt.Errorf("the peer's models could not be read: %s", resp.Status))
}
cluster := resp.Header.Get(ClusterHeader)
if cluster == "" {
return fail(errors.New("the peer does not say its cluster id: is it an llmgw gateway of this version or later?"))
}
var list struct {
Data []struct {
ID string `json:"id"`
Description string `json:"description"`
ContextLength int `json:"context_length"`
MaxOutput int `json:"max_output_tokens"`
Capabilities catalog.Capabilities `json:"capabilities"`
AliasOf string `json:"alias_of"`
Peer string `json:"peer"`
} `json:"data"`
}
if err := json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&list); err != nil {
return fail(fmt.Errorf("the peer's model list is not what an llmgw gateway answers: %w", err))
}
for _, m := range list.Data {
// Not an alias, which is a second name for a model already here,
// nor a model the peer itself has from a peer of its own: federating
// those would grow a name a hop, A/B/A/..., each time two clusters
// read each other's lists.
if m.AliasOf != "" || m.Peer != "" || !validModelID(m.ID) {
continue
}
res.models = append(res.models, PeerModel{ID: m.ID, Description: m.Description,
ContextLength: m.ContextLength, MaxOutput: m.MaxOutput, Capabilities: m.Capabilities})
}
slices.SortFunc(res.models, func(a, b PeerModel) int { return strings.Compare(a.ID, b.ID) })
res.cluster = cluster
res.health.Up = true
return res
}
// validModelID is a model name this cluster can carry: printable, no spaces,
// and short enough to be a name.
func validModelID(id string) bool {
if id == "" || len(id) > 200 {
return false
}
return !strings.ContainsFunc(id, func(r rune) bool { return r <= ' ' || r == 0x7f })
}
// Run checks every peer at once each interval until ctx ends, the first
// time straight away.
func (s *Service) Run(ctx context.Context) {
t := time.NewTicker(s.cfg.Interval)
defer t.Stop()
for {
s.SyncAll(ctx)
select {
case <-ctx.Done():
return
case <-t.C:
}
}
}
// Status is a peer as `llmgw ctl federation list` shows it.
type Status struct {
Name string `json:"name"`
URL string `json:"url"`
KeyFile string `json:"key_file,omitempty"`
SendTags bool `json:"send_tags"`
// Cluster is the peer's own cluster id.
Cluster string `json:"cluster,omitempty"`
Health
// LatencyMs is Latency in milliseconds, for reading.
LatencyMs float64 `json:"latency_ms"`
// SyncedAt is when its models last changed here.
SyncedAt time.Time `json:"synced_at,omitempty"`
Models []ModelStatus `json:"models"`
CreatedAt time.Time `json:"created_at"`
CreatedBy string `json:"created_by,omitempty"`
}
// ModelStatus is one of a peer's models: its name here, the peer's own,
// and whether the peer has stopped offering it.
type ModelStatus struct {
Name string `json:"name"`
Upstream string `json:"upstream"`
Withdrawn bool `json:"withdrawn,omitempty"`
}
// Status is one peer; false when there is none of the name.
func (s *Service) Status(name string) (Status, bool) {
s.mu.Lock()
defer s.mu.Unlock()
p, ok := s.peers[name]
if !ok {
return Status{}, false
}
return statusOf(p), true
}
// List is every peer, by name.
func (s *Service) List() []Status {
s.mu.Lock()
defer s.mu.Unlock()
out := make([]Status, 0, len(s.peers))
for _, n := range s.namesLocked() {
out = append(out, statusOf(s.peers[n]))
}
return out
}
func statusOf(p *peer) Status {
st := Status{Name: p.rec.Name, URL: p.rec.URL, KeyFile: p.rec.KeyFile, SendTags: p.rec.SendTags,
Cluster: p.rec.Cluster, Health: p.health, LatencyMs: float64(p.health.Latency.Microseconds()) / 1000,
SyncedAt: p.rec.SyncedAt, CreatedAt: p.rec.CreatedAt, CreatedBy: p.rec.CreatedBy,
Models: make([]ModelStatus, 0, len(p.models))}
for _, m := range p.models {
st.Models = append(st.Models, ModelStatus{Name: p.rec.Name + "/" + m.ID, Upstream: m.ID, Withdrawn: m.Withdrawn})
}
return st
}
+42
View File
@@ -0,0 +1,42 @@
package federation
import "github.com/prometheus/client_golang/prometheus"
var (
upDesc = prometheus.NewDesc("llmgw_federation_peer_up",
"Whether a federated peer answered its last check: 1 up, 0 down and not sent requests.",
[]string{"peer"}, nil)
latencyDesc = prometheus.NewDesc("llmgw_federation_peer_latency_seconds",
"How long a federated peer took to answer /ready at its last check.",
[]string{"peer"}, nil)
modelsDesc = prometheus.NewDesc("llmgw_federation_peer_models",
"How many models a federated peer offers this cluster.",
[]string{"peer"}, nil)
)
// Describe and Collect publish each peer's health beside the traffic it
// takes, so "overflow went to the other site" and "the other site was slow"
// are on one chart.
func (s *Service) Describe(ch chan<- *prometheus.Desc) {
ch <- upDesc
ch <- latencyDesc
ch <- modelsDesc
}
func (s *Service) Collect(ch chan<- prometheus.Metric) {
for _, st := range s.List() {
up := 0.0
if st.Up {
up = 1
}
offered := 0
for _, m := range st.Models {
if !m.Withdrawn {
offered++
}
}
ch <- prometheus.MustNewConstMetric(upDesc, prometheus.GaugeValue, up, st.Name)
ch <- prometheus.MustNewConstMetric(latencyDesc, prometheus.GaugeValue, st.Latency.Seconds(), st.Name)
ch <- prometheus.MustNewConstMetric(modelsDesc, prometheus.GaugeValue, float64(offered), st.Name)
}
}
+90 -4
View File
@@ -51,6 +51,9 @@ type Service struct {
// SetCheck.
check func(context.Context, catalog.Model) error
// federated are the models federated clusters offer; see SetFederated.
federated func() []catalog.Model
// mu serialises rebuilds. They are rare — a person adding a model — and two
// at once could apply catalogs in either order, leaving the one in force
// disagreeing with the database that outlives it.
@@ -128,7 +131,7 @@ func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error {
if err != nil {
return err
}
next := append([]catalog.Model{}, s.declared...)
next := s.base(stored)
for _, e := range stored {
if e.Name == m.Name {
continue // this call replaces it
@@ -174,7 +177,7 @@ func (s *Service) Remove(ctx context.Context, name string) error {
if err != nil {
return err
}
next := append([]catalog.Model{}, s.declared...)
next := s.base(stored)
found := false
for _, e := range stored {
if e.Name == name {
@@ -371,7 +374,7 @@ func (s *Service) update(ctx context.Context, name, by, what string, change func
if err != nil {
return catalog.Model{}, err
}
next := append([]catalog.Model{}, s.declared...)
next := s.base(stored)
found := false
for _, e := range stored {
if e.Name == name {
@@ -416,12 +419,19 @@ func (s *Service) Load(ctx context.Context) (*catalog.Catalog, error) {
s.mu.Lock()
defer s.mu.Unlock()
return s.load(ctx)
}
// load is Load, with the lock held.
func (s *Service) load(ctx context.Context) (*catalog.Catalog, error) {
stored, err := s.storedModels(ctx)
if err != nil {
return nil, err
}
next := append([]catalog.Model{}, s.declared...)
// The federated models before the stored ones, so a stored model that
// falls back to a peer's is checked against a catalog that has it.
next := s.base(stored)
for _, m := range stored {
// Checked one at a time and against what is already accepted, so one
// bad model cannot take a good one with it.
@@ -459,6 +469,82 @@ func (s *Service) storedModels(ctx context.Context) ([]catalog.Model, error) {
return out, nil
}
// SetFederated supplies the models federated clusters offer, which join the
// catalog after the declared ones and before those added at runtime.
//
// A function rather than a list, read at every rebuild, because the list
// changes whenever a peer's does; Refresh is how federation says it has.
func (s *Service) SetFederated(f func() []catalog.Model) {
s.mu.Lock()
defer s.mu.Unlock()
s.federated = f
}
// Refresh rebuilds the catalog and puts it in force: after a peer's models
// changed. Built as at startup, so a stored model that no longer makes a
// valid catalog is skipped and said rather than taking the rest with it.
func (s *Service) Refresh(ctx context.Context) error {
s.mu.Lock()
defer s.mu.Unlock()
built, err := s.load(ctx)
if err != nil {
return err
}
s.apply.ReplaceCatalog(ctx, built)
return nil
}
// base is the declared models, then the federated ones whose names nothing
// declared or added at runtime already uses.
//
// A peer's model is named PEER/MODEL, which a local model could also be
// called — Hugging Face ids look just like that. The local one wins, for the
// reason configuration wins over the database: a request for a model this
// cluster runs must not leave it because a peer started offering a name.
func (s *Service) base(stored []catalog.Model) []catalog.Model {
next := append([]catalog.Model{}, s.declared...)
if s.federated == nil {
return next
}
taken := map[string]bool{}
for _, list := range [][]catalog.Model{s.declared, stored} {
for _, m := range list {
taken[m.Name] = true
for _, a := range m.Aliases {
taken[a] = true
}
}
}
for _, f := range s.federated() {
if taken[f.Name] {
s.log.Warn("a federated model has the name of a model of this cluster's; not offered",
"model", f.Name)
continue
}
next = append(next, f)
}
return next
}
// SetFallbacks replaces the models a model added at runtime falls back to
// when it has no capacity: other models of the catalog, a federated peer's
// among them.
func (s *Service) SetFallbacks(ctx context.Context, name string, fallbacks []string, by string) (catalog.Model, error) {
final, err := s.update(ctx, name, by, "change its fallbacks", func(m *catalog.Model) error {
for _, f := range fallbacks {
if f == m.Name {
return fmt.Errorf("model %q cannot fall back to itself", name)
}
}
m.Fallbacks = fallbacks
return nil
})
if err == nil {
s.log.Info("model fallbacks set", "model", name, "fallbacks", fallbacks, "by", by)
}
return final, err
}
// SetCheck runs before a model is added, and refuses it by returning an error.
// Used to catch weights the engine cannot load, which otherwise are placed and
// then fail on every node they are tried on.