Federation: another llmgw cluster as a provider, its models as PEER/MODEL
ctl federation add/list/remove/sync keeps peers in the store (migration 0029); their /v1/models are offered here as external models named PEER/MODEL and kept in step, a model still named by a fallback kept withdrawn. Peers are checked at /ready, timed, and taken out of routing while down. Requests to a peer carry x-llmgw-hop and are refused (508) when they come back to a cluster they passed through; residency and no_store keep requests off peers; X-Request-Id and, with --send-tags, attribution tags go along. ctl model fallbacks sets overflow to a peer. Console view, metrics, admin API routes and docs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
40e9fd2ab1
33 files changed
+2944
-23
No files matched your search
@@ -32,6 +32,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
@@ -58,6 +59,9 @@ type API struct {
|
||||
history History
|
||||
// audit reads the audit log.
|
||||
audit AuditReader
|
||||
// federation is the other llmgw clusters whose models are offered here;
|
||||
// nil without a store.
|
||||
federation *federation.Service
|
||||
// guard is the guardrail policies' library and attachments.
|
||||
guard *GuardAdmin
|
||||
// revoke records a removed node, so its certificate is refused.
|
||||
@@ -172,6 +176,7 @@ func (a *API) Register(mux Mux) {
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel)
|
||||
a.registerCanary(mux)
|
||||
a.registerEval(mux)
|
||||
a.registerFederation(mux)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/origin", a.setOrigin)
|
||||
mux.HandleFunc("GET /api/v1/plan", a.planModel)
|
||||
mux.HandleFunc("POST /api/v1/metrics/range", a.metricsBatch)
|
||||
@@ -1810,6 +1815,9 @@ func externalLabel(m catalog.Model) string {
|
||||
if !m.IsExternal() {
|
||||
return ""
|
||||
}
|
||||
if m.IsPeer() {
|
||||
return "peer " + m.External.Peer + " · " + m.External.Model
|
||||
}
|
||||
host := m.External.URL
|
||||
if u, err := url.Parse(host); err == nil && u.Host != "" {
|
||||
host = u.Host
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
|
||||
)
|
||||
|
||||
// Federation: other llmgw clusters whose models this one offers, as
|
||||
// PEER/MODEL, and the fallbacks that send overflow to them.
|
||||
|
||||
// SetFederation gives the API the peers to list and change.
|
||||
func (a *API) SetFederation(f *federation.Service) { a.federation = f }
|
||||
|
||||
// FederationView is this cluster's id and its peers.
|
||||
type FederationView struct {
|
||||
// Cluster is this cluster's own id, which its peers see in the hop
|
||||
// header.
|
||||
Cluster string `json:"cluster"`
|
||||
Peers []federation.Status `json:"peers"`
|
||||
}
|
||||
|
||||
// FallbacksRequest replaces a model's fallbacks; empty takes them all away.
|
||||
type FallbacksRequest struct {
|
||||
Fallbacks []string `json:"fallbacks"`
|
||||
}
|
||||
|
||||
// FallbacksView is a model's fallbacks, as they are now.
|
||||
type FallbacksView struct {
|
||||
Model string `json:"model"`
|
||||
Fallbacks []string `json:"fallbacks"`
|
||||
}
|
||||
|
||||
type fallbackSetter interface {
|
||||
SetFallbacks(ctx context.Context, name string, fallbacks []string, by string) (catalog.Model, error)
|
||||
}
|
||||
|
||||
func (a *API) registerFederation(mux Mux) {
|
||||
need := func(w http.ResponseWriter) bool {
|
||||
if a.federation == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store to keep federation peers in")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
mux.HandleFunc("GET /api/v1/federation", func(w http.ResponseWriter, _ *http.Request) {
|
||||
if !need(w) {
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, FederationView{Cluster: a.federation.Self(), Peers: a.federation.List()})
|
||||
})
|
||||
mux.HandleFunc("POST /api/v1/federation", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !need(w) {
|
||||
return
|
||||
}
|
||||
var req federation.AddRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<14)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
st, err := a.federation.Add(r.Context(), req, actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, st)
|
||||
})
|
||||
mux.HandleFunc("DELETE /api/v1/federation/{peer}", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !need(w) {
|
||||
return
|
||||
}
|
||||
name := r.PathValue("peer")
|
||||
if err := a.federation.Remove(r.Context(), name); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, Removed{Removed: name})
|
||||
})
|
||||
mux.HandleFunc("POST /api/v1/federation/{peer}/sync", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !need(w) {
|
||||
return
|
||||
}
|
||||
st, err := a.federation.Sync(r.Context(), r.PathValue("peer"))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, st)
|
||||
})
|
||||
mux.HandleFunc("PUT /api/v1/models/{name}/fallbacks", func(w http.ResponseWriter, r *http.Request) {
|
||||
setter, ok := a.models.(fallbackSetter)
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store to keep the setting in")
|
||||
return
|
||||
}
|
||||
var req FallbacksRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<14)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
var fbs []string
|
||||
for _, f := range req.Fallbacks {
|
||||
if f = strings.TrimSpace(f); f != "" {
|
||||
fbs = append(fbs, f)
|
||||
}
|
||||
}
|
||||
m, err := setter.SetFallbacks(r.Context(), r.PathValue("name"), fbs, actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
out := m.Fallbacks
|
||||
if out == nil {
|
||||
out = []string{}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, FallbacksView{Model: m.Name, Fallbacks: out})
|
||||
})
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
@@ -56,13 +57,14 @@ type Param struct {
|
||||
// PathParams describes the parameters in Routes' paths, by the segment
|
||||
// before them and their name.
|
||||
var PathParams = map[string]string{
|
||||
"models/{name}": "The model's name in the catalog.",
|
||||
"nodes/{id}": "The node's ID, as `llmgw ctl nodes` lists it.",
|
||||
"jobs/{id}": "The job's ID, as starting it answered.",
|
||||
"runs/{id}": "The quality run's ID, as starting it answered.",
|
||||
"requests/{id}": "The request's ID: the x-request-id a client was answered with.",
|
||||
"fabrics/{id}": "The physical fabric's ID, as GET /api/v1/fabrics lists it.",
|
||||
"quotas/{scope}": "The scope: org:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT, or their IDs after the colon.",
|
||||
"models/{name}": "The model's name in the catalog.",
|
||||
"nodes/{id}": "The node's ID, as `llmgw ctl nodes` lists it.",
|
||||
"jobs/{id}": "The job's ID, as starting it answered.",
|
||||
"runs/{id}": "The quality run's ID, as starting it answered.",
|
||||
"requests/{id}": "The request's ID: the x-request-id a client was answered with.",
|
||||
"fabrics/{id}": "The physical fabric's ID, as GET /api/v1/fabrics lists it.",
|
||||
"quotas/{scope}": "The scope: org:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT, or their IDs after the colon.",
|
||||
"federation/{peer}": "The federation peer's name, as `llmgw ctl federation list` lists it.",
|
||||
}
|
||||
|
||||
// Routes is every endpoint Register mounts.
|
||||
@@ -210,6 +212,21 @@ var Routes = []Route{
|
||||
{Method: "DELETE", Path: "/api/v1/models/{name}/eval-gate", Summary: "Stop holding a model's updates to a suite",
|
||||
Response: EvalGateView{}},
|
||||
|
||||
// Federation: other llmgw clusters as providers.
|
||||
{Method: "GET", Path: "/api/v1/federation", Summary: "This cluster's id, and its federation peers with their health and models",
|
||||
Response: FederationView{}},
|
||||
{Method: "POST", Path: "/api/v1/federation", Summary: "Add another llmgw cluster as a peer, or change one",
|
||||
Description: "The peer is checked first: its /ready, timed, and its /v1/models with the key. Its models join the catalog as PEER/MODEL.",
|
||||
Request: federation.AddRequest{}, Response: federation.Status{}, Status: http.StatusCreated},
|
||||
{Method: "DELETE", Path: "/api/v1/federation/{peer}", Summary: "Remove a federation peer and its models",
|
||||
Description: "Refused while a model here falls back to one of the peer's.",
|
||||
Response: Removed{}},
|
||||
{Method: "POST", Path: "/api/v1/federation/{peer}/sync", Summary: "Check a peer now, rather than at the next interval",
|
||||
Response: federation.Status{}},
|
||||
{Method: "PUT", Path: "/api/v1/models/{name}/fallbacks", Summary: "Replace the models a model falls back to when it has no capacity",
|
||||
Description: "A model added at runtime only. A federation peer's model, PEER/MODEL, sends the overflow to the other cluster.",
|
||||
Request: FallbacksRequest{}, Response: FallbacksView{}},
|
||||
|
||||
// Tenancy: quotas and guardrails.
|
||||
{Method: "GET", Path: "/api/v1/quotas", Summary: "Every capacity quota, with what its scope holds now",
|
||||
Response: []QuotaView{}},
|
||||
|
||||
@@ -1017,8 +1017,16 @@ type External struct {
|
||||
// rather than the key itself, so the key is in neither the catalog, the
|
||||
// store nor the audit log; empty for a provider that asks for none.
|
||||
APIKeyFile string `json:"api_key_file,omitempty" koanf:"api_key_file"`
|
||||
// Peer names the federated llmgw cluster this model is offered by, for
|
||||
// a model that federation added (`llmgw ctl federation add`) rather
|
||||
// than one written by hand. Such a model is named PEER/MODEL, comes and
|
||||
// goes with the peer's own list, and is sent only while the peer is up.
|
||||
Peer string `json:"peer,omitempty" koanf:"peer"`
|
||||
}
|
||||
|
||||
// IsPeer reports whether a model is offered by a federated cluster.
|
||||
func (m Model) IsPeer() bool { return m.External != nil && m.External.Peer != "" }
|
||||
|
||||
func (e *External) validate(model string, profiles int) error {
|
||||
if profiles > 0 {
|
||||
return fmt.Errorf("model %q: an external model is served elsewhere and has no profiles", model)
|
||||
|
||||
@@ -87,6 +87,7 @@ type Controller struct {
|
||||
disp reconciler.Dispatcher
|
||||
sink EndpointSink
|
||||
usage UsageSource
|
||||
peers Peers
|
||||
|
||||
mu sync.Mutex
|
||||
last string // fingerprint of the last published list
|
||||
@@ -145,6 +146,27 @@ func (c *Controller) collectUsage() {
|
||||
}
|
||||
}
|
||||
|
||||
// Peers says which federated clusters' models may be sent requests now.
|
||||
//
|
||||
// Federation keeps it: whether each peer answered its last check, and
|
||||
// whether it still offers the model. A peer model is in the catalog either
|
||||
// way — a local model may name it as a fallback — and is published only
|
||||
// while this says yes, which is what takes a peer that is down out of
|
||||
// routing within a publish.
|
||||
type Peers interface {
|
||||
// PeerRoute is the peer's own cluster id and whether requests sent to
|
||||
// it carry tags, and false when the model is not to be sent anything.
|
||||
PeerRoute(peer, upstreamModel string) (cluster string, sendTags, ok bool)
|
||||
}
|
||||
|
||||
// SetPeers supplies the federation's view of its peers. Without one, no
|
||||
// peer model is published.
|
||||
func (c *Controller) SetPeers(p Peers) {
|
||||
c.mu.Lock()
|
||||
c.peers = p
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// LoadSource reports each model's replicas' load, keyed by catalog name.
|
||||
type LoadSource interface {
|
||||
ModelLoad() map[string]reconciler.Load
|
||||
@@ -349,11 +371,25 @@ func (c *Controller) endpoints() []endpoint.Info {
|
||||
// one each, with the provider's key read from its file every time, so a key
|
||||
// rotated on disk is used from the next publish on.
|
||||
func (c *Controller) externalEndpoints() []endpoint.Info {
|
||||
c.mu.Lock()
|
||||
peers := c.peers
|
||||
c.mu.Unlock()
|
||||
var out []endpoint.Info
|
||||
for _, m := range c.cat.Load().Models() {
|
||||
if !m.IsExternal() {
|
||||
continue
|
||||
}
|
||||
var cluster string
|
||||
var tags bool
|
||||
if m.IsPeer() {
|
||||
ok := false
|
||||
if peers != nil {
|
||||
cluster, tags, ok = peers.PeerRoute(m.External.Peer, m.External.Model)
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
key := ""
|
||||
if f := m.External.APIKeyFile; f != "" {
|
||||
b, err := secrets.Read(context.Background(), f)
|
||||
@@ -363,11 +399,18 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
|
||||
}
|
||||
key = strings.TrimSpace(string(b))
|
||||
}
|
||||
// A peer's endpoint says so in its id, which is what usage records
|
||||
// as the replica that answered.
|
||||
id := "external:" + m.Name
|
||||
if m.IsPeer() {
|
||||
id = "peer:" + m.Name
|
||||
}
|
||||
out = append(out, endpoint.Info{
|
||||
ID: "external:" + m.Name, URL: strings.TrimRight(m.External.URL, "/"),
|
||||
ID: id, URL: strings.TrimRight(m.External.URL, "/"),
|
||||
Models: append([]string{m.Name}, m.Aliases...), APIKey: key,
|
||||
Engine: endpoint.EngineExternal, Pool: "external", Weight: 1,
|
||||
UpstreamModel: m.External.Model,
|
||||
Peer: m.External.Peer, PeerCluster: cluster, PeerTags: tags,
|
||||
})
|
||||
}
|
||||
return out
|
||||
@@ -377,8 +420,8 @@ func (c *Controller) externalEndpoints() []endpoint.Info {
|
||||
func fingerprint(infos []endpoint.Info) string {
|
||||
var b strings.Builder
|
||||
for _, i := range infos {
|
||||
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d|%s|%x;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority,
|
||||
i.UpstreamModel, sha256.Sum256([]byte(i.APIKey)))
|
||||
fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d|%s|%s|%s|%t|%x;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight,
|
||||
i.Priority, i.UpstreamModel, i.Peer, i.PeerCluster, i.PeerTags, sha256.Sum256([]byte(i.APIKey)))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,661 @@
|
||||
// Package federation makes other llmgw clusters providers of this one.
|
||||
//
|
||||
// A peer is another llmgw gateway, reached with a key it issued to this
|
||||
// cluster. Its models are offered here as external models named PEER/MODEL,
|
||||
// kept in step with its own /v1/models, and sent requests only while its
|
||||
// /ready answers. A local model names one as a fallback to send overflow to
|
||||
// the other site.
|
||||
//
|
||||
// Always PEER/MODEL, never the peer's own name for a model, even when nothing
|
||||
// here uses that name. A name that left the cluster or stayed depending on
|
||||
// what the catalog happened to hold would move traffic between sites when
|
||||
// somebody added an unrelated model; the prefix also says, on every usage row
|
||||
// and every x-llmgw-served-model header, which site answered.
|
||||
//
|
||||
// Everything that keeps a request inside the cluster already applies: a peer
|
||||
// model is an external model, so a project whose requests stay in the
|
||||
// cluster, or a policy that keeps nothing, is never sent to one.
|
||||
package federation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/secrets"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// ClusterHeader is the response header in which a gateway says its own
|
||||
// cluster id, on /v1/models. Federation reads a peer's from it, to know the
|
||||
// peer in a request's hop header.
|
||||
const ClusterHeader = "x-llmgw-cluster"
|
||||
|
||||
// Store is the persistence federation needs.
|
||||
type Store interface {
|
||||
PutFederationPeer(ctx context.Context, p store.FederationPeer) error
|
||||
SetFederationPeerModels(ctx context.Context, name, cluster string, models json.RawMessage, at time.Time) error
|
||||
DeleteFederationPeer(ctx context.Context, name string) error
|
||||
FederationPeers(ctx context.Context) ([]store.FederationPeer, error)
|
||||
}
|
||||
|
||||
// Catalog is what puts a changed peer list in force: the model service.
|
||||
type Catalog interface {
|
||||
Refresh(ctx context.Context) error
|
||||
}
|
||||
|
||||
// Config tunes federation.
|
||||
type Config struct {
|
||||
// Self is this cluster's id. A peer that says the same is this cluster,
|
||||
// reached by another address, and is refused.
|
||||
Self string
|
||||
// Interval is how often each peer is checked; zero is 10s.
|
||||
Interval time.Duration
|
||||
// Timeout bounds one check of one peer; zero is 5s.
|
||||
Timeout time.Duration
|
||||
// Client makes the checks; nil is a plain one.
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
// PeerModel is what a peer says about one model it offers.
|
||||
type PeerModel struct {
|
||||
// ID is the peer's name for it, and what it is asked for.
|
||||
ID string `json:"id"`
|
||||
Description string `json:"description,omitempty"`
|
||||
ContextLength int `json:"context_length,omitempty"`
|
||||
MaxOutput int `json:"max_output,omitempty"`
|
||||
Capabilities catalog.Capabilities `json:"capabilities,omitempty"`
|
||||
// Withdrawn is a model the peer no longer offers, kept in the catalog
|
||||
// because a model here still falls back to it: removing it would make
|
||||
// that model invalid. Nothing is sent to it, and it goes when nothing
|
||||
// names it any more.
|
||||
Withdrawn bool `json:"withdrawn,omitempty"`
|
||||
}
|
||||
|
||||
// Service keeps the peers, their health and their models.
|
||||
type Service struct {
|
||||
cfg Config
|
||||
store Store
|
||||
catalog Catalog
|
||||
// current is the catalog in force, read to tell which of a peer's
|
||||
// models a local model names.
|
||||
current func() *catalog.Catalog
|
||||
log *slog.Logger
|
||||
|
||||
// op serialises what changes the peer set or a peer's models, so two
|
||||
// rebuilds cannot land in the wrong order.
|
||||
op sync.Mutex
|
||||
|
||||
mu sync.Mutex
|
||||
peers map[string]*peer
|
||||
}
|
||||
|
||||
type peer struct {
|
||||
rec store.FederationPeer
|
||||
models []PeerModel
|
||||
health Health
|
||||
}
|
||||
|
||||
// Health is how a peer answered its last check.
|
||||
type Health struct {
|
||||
Up bool `json:"up"`
|
||||
// Checked is when; zero before the first check.
|
||||
Checked time.Time `json:"checked,omitempty"`
|
||||
// Latency is how long the peer took to answer /ready: the network to
|
||||
// the other site and the peer's front, not a model's speed.
|
||||
Latency time.Duration `json:"latency_ns,omitempty"`
|
||||
// Error is why the peer is down, empty while it is up.
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// New builds the service. Load reads the peers kept in the store.
|
||||
func New(cfg Config, st Store, cat Catalog, current func() *catalog.Catalog, log *slog.Logger) *Service {
|
||||
if cfg.Interval <= 0 {
|
||||
cfg.Interval = 10 * time.Second
|
||||
}
|
||||
if cfg.Timeout <= 0 {
|
||||
cfg.Timeout = 5 * time.Second
|
||||
}
|
||||
if cfg.Client == nil {
|
||||
cfg.Client = &http.Client{}
|
||||
}
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
return &Service{cfg: cfg, store: st, catalog: cat, current: current, log: log, peers: map[string]*peer{}}
|
||||
}
|
||||
|
||||
// Self is this cluster's id.
|
||||
func (s *Service) Self() string { return s.cfg.Self }
|
||||
|
||||
// Load reads the peers and the models they last offered from the store, so
|
||||
// the catalog built at startup has them before any peer is asked again.
|
||||
// Every peer reads as down until its first check.
|
||||
func (s *Service) Load(ctx context.Context) error {
|
||||
recs, err := s.store.FederationPeers(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, r := range recs {
|
||||
var models []PeerModel
|
||||
if err := json.Unmarshal(r.Models, &models); err != nil {
|
||||
s.log.Warn("a federation peer's models could not be read; they come back at its next check",
|
||||
"peer", r.Name, "err", err)
|
||||
}
|
||||
s.peers[r.Name] = &peer{rec: r, models: models, health: Health{Error: "not checked yet"}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Models are the peers' models as catalog entries, withdrawn ones included:
|
||||
// what the model service adds to the catalog.
|
||||
func (s *Service) Models() []catalog.Model {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []catalog.Model
|
||||
for _, name := range s.namesLocked() {
|
||||
p := s.peers[name]
|
||||
for _, m := range p.models {
|
||||
out = append(out, catalogModel(p.rec, m))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Service) namesLocked() []string {
|
||||
names := make([]string, 0, len(s.peers))
|
||||
for n := range s.peers {
|
||||
names = append(names, n)
|
||||
}
|
||||
slices.Sort(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// catalogModel is a peer's model as an entry of this catalog.
|
||||
func catalogModel(rec store.FederationPeer, m PeerModel) catalog.Model {
|
||||
desc := "on " + rec.Name
|
||||
if m.Description != "" {
|
||||
desc += ": " + m.Description
|
||||
}
|
||||
return catalog.Model{
|
||||
Name: rec.Name + "/" + m.ID, Source: "peer:" + rec.Name, Description: desc,
|
||||
ContextLength: m.ContextLength, MaxOutput: m.MaxOutput, Capabilities: m.Capabilities,
|
||||
External: &catalog.External{URL: rec.URL, Model: m.ID, APIKeyFile: rec.KeyFile, Peer: rec.Name},
|
||||
}
|
||||
}
|
||||
|
||||
// PeerRoute says whether a peer's model may be sent requests now: the peer
|
||||
// answered its last check and still offers it. The controller publishes the
|
||||
// model's endpoint only while it does.
|
||||
func (s *Service) PeerRoute(name, model string) (cluster string, sendTags, ok bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
p, found := s.peers[name]
|
||||
if !found || !p.health.Up {
|
||||
return "", false, false
|
||||
}
|
||||
for _, m := range p.models {
|
||||
if m.ID == model {
|
||||
return p.rec.Cluster, p.rec.SendTags, !m.Withdrawn
|
||||
}
|
||||
}
|
||||
return "", false, false
|
||||
}
|
||||
|
||||
// peerName is what a peer may be called: the prefix of its models' names,
|
||||
// and a tag value, so letters, digits and a few separators, and no slash.
|
||||
var peerName = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]{0,31}$`)
|
||||
|
||||
// AddRequest adds a peer.
|
||||
type AddRequest struct {
|
||||
// Name is what this cluster calls the peer, and the prefix of its
|
||||
// models: NAME/MODEL.
|
||||
Name string `json:"name"`
|
||||
// URL is the peer gateway's client address, without /v1.
|
||||
URL string `json:"url"`
|
||||
// KeyFile is a file on the control plane holding a key the peer issued
|
||||
// to this cluster (or env:NAME, vault:PATH#FIELD).
|
||||
KeyFile string `json:"key_file"`
|
||||
// SendTags makes requests sent to the peer carry x-llmgw-tags naming
|
||||
// this cluster and the sending project. Off unless asked: the peer
|
||||
// learns who in this cluster is asking.
|
||||
SendTags bool `json:"send_tags,omitempty"`
|
||||
}
|
||||
|
||||
// Add checks a peer and keeps it, its models joining the catalog.
|
||||
//
|
||||
// Checked before anything is kept: a mistyped address or a key the peer
|
||||
// refuses is said now, rather than found as a peer that is always down.
|
||||
func (s *Service) Add(ctx context.Context, req AddRequest, by string) (Status, error) {
|
||||
req.Name = strings.TrimSpace(req.Name)
|
||||
req.URL = strings.TrimRight(strings.TrimSpace(req.URL), "/")
|
||||
req.URL = strings.TrimSuffix(req.URL, "/v1")
|
||||
if !peerName.MatchString(req.Name) {
|
||||
return Status{}, fmt.Errorf("peer name %q: 1 to 32 lower-case letters, digits, '_', '.' or '-', starting with a letter or digit", req.Name)
|
||||
}
|
||||
u, err := url.Parse(req.URL)
|
||||
if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" {
|
||||
return Status{}, fmt.Errorf("peer address %q is not an http(s) address", req.URL)
|
||||
}
|
||||
if req.KeyFile == "" {
|
||||
return Status{}, errors.New("give the file holding the key the peer issued to this cluster (--key-file)")
|
||||
}
|
||||
if cur := s.current(); cur != nil {
|
||||
for _, m := range cur.Models() {
|
||||
if !m.IsPeer() && strings.HasPrefix(m.Name, req.Name+"/") {
|
||||
return Status{}, fmt.Errorf("model %q of this cluster already starts with %s/; call the peer something else",
|
||||
m.Name, req.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
s.op.Lock()
|
||||
defer s.op.Unlock()
|
||||
|
||||
rec := store.FederationPeer{Name: req.Name, URL: req.URL, KeyFile: req.KeyFile, SendTags: req.SendTags,
|
||||
CreatedAt: time.Now(), CreatedBy: by}
|
||||
s.mu.Lock()
|
||||
old, existed := s.peers[req.Name]
|
||||
if existed {
|
||||
rec.CreatedAt, rec.CreatedBy = old.rec.CreatedAt, old.rec.CreatedBy
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
res := s.check(ctx, rec)
|
||||
if res.err != nil {
|
||||
return Status{}, fmt.Errorf("peer %s: %w", req.Name, res.err)
|
||||
}
|
||||
if res.cluster != "" && res.cluster == s.cfg.Self {
|
||||
return Status{}, fmt.Errorf("%s is this cluster: its gateway answers with this cluster's own id", req.URL)
|
||||
}
|
||||
rec.Cluster, rec.SyncedAt = res.cluster, time.Now()
|
||||
models := res.models
|
||||
if existed {
|
||||
models = s.keepReferenced(old, res.models)
|
||||
}
|
||||
b, _ := json.Marshal(models)
|
||||
rec.Models = b
|
||||
if err := s.store.PutFederationPeer(ctx, rec); err != nil {
|
||||
return Status{}, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.peers[req.Name] = &peer{rec: rec, models: models, health: res.health}
|
||||
s.mu.Unlock()
|
||||
if err := s.catalog.Refresh(ctx); err != nil {
|
||||
return Status{}, fmt.Errorf("the peer is kept, but the catalog could not be rebuilt: %w", err)
|
||||
}
|
||||
s.log.Info("federation peer added", "peer", req.Name, "url", req.URL, "models", len(models), "by", by)
|
||||
st, _ := s.Status(req.Name)
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// Remove takes a peer and its models away.
|
||||
//
|
||||
// Refused while a model here falls back to one of them, or has one as its
|
||||
// canary, for the reason removing a model is: a fallback pointing at
|
||||
// nothing is found during the outage it was meant to cover.
|
||||
func (s *Service) Remove(ctx context.Context, name string) error {
|
||||
s.op.Lock()
|
||||
defer s.op.Unlock()
|
||||
|
||||
s.mu.Lock()
|
||||
p, ok := s.peers[name]
|
||||
s.mu.Unlock()
|
||||
if !ok {
|
||||
return fmt.Errorf("no federation peer called %q: `llmgw ctl federation list` lists them", name)
|
||||
}
|
||||
var users []string
|
||||
if cur := s.current(); cur != nil {
|
||||
for _, m := range cur.Models() {
|
||||
if m.IsPeer() {
|
||||
continue
|
||||
}
|
||||
for _, ref := range references(m) {
|
||||
if strings.HasPrefix(ref, name+"/") {
|
||||
users = append(users, m.Name+" → "+ref)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(users) > 0 {
|
||||
return fmt.Errorf("models here still use %s's: %s; change their fallbacks first (`llmgw ctl model fallbacks`)",
|
||||
name, strings.Join(users, ", "))
|
||||
}
|
||||
if err := s.store.DeleteFederationPeer(ctx, name); err != nil && !errors.Is(err, store.ErrNotFound) {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
delete(s.peers, name)
|
||||
s.mu.Unlock()
|
||||
if err := s.catalog.Refresh(ctx); err != nil {
|
||||
return fmt.Errorf("the peer is removed, but the catalog could not be rebuilt: %w", err)
|
||||
}
|
||||
s.log.Info("federation peer removed", "peer", name, "models", len(p.models))
|
||||
return nil
|
||||
}
|
||||
|
||||
// references are the other models a model names: its fallbacks and its
|
||||
// canary.
|
||||
func references(m catalog.Model) []string {
|
||||
out := slices.Clone(m.Fallbacks)
|
||||
if m.Canary != nil {
|
||||
out = append(out, m.Canary.Model)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// keepReferenced is what a peer offers now, plus those it offered before and
|
||||
// no longer does that a model here still names, kept withdrawn.
|
||||
func (s *Service) keepReferenced(old *peer, offered []PeerModel) []PeerModel {
|
||||
out := slices.Clone(offered)
|
||||
named := map[string]bool{}
|
||||
if cur := s.current(); cur != nil {
|
||||
for _, m := range cur.Models() {
|
||||
if m.IsPeer() {
|
||||
continue
|
||||
}
|
||||
for _, ref := range references(m) {
|
||||
named[ref] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, m := range old.models {
|
||||
if slices.ContainsFunc(offered, func(o PeerModel) bool { return o.ID == m.ID }) {
|
||||
continue
|
||||
}
|
||||
if named[old.rec.Name+"/"+m.ID] {
|
||||
m.Withdrawn = true
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Sync checks one peer now, rather than at the next interval.
|
||||
func (s *Service) Sync(ctx context.Context, name string) (Status, error) {
|
||||
s.mu.Lock()
|
||||
p, ok := s.peers[name]
|
||||
var rec store.FederationPeer
|
||||
if ok {
|
||||
rec = p.rec
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if !ok {
|
||||
return Status{}, fmt.Errorf("no federation peer called %q: `llmgw ctl federation list` lists them", name)
|
||||
}
|
||||
res := s.check(ctx, rec)
|
||||
s.apply(ctx, map[string]checkResult{name: res})
|
||||
st, _ := s.Status(name)
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// SyncAll checks every peer at once, and puts what changed in force.
|
||||
func (s *Service) SyncAll(ctx context.Context) {
|
||||
s.mu.Lock()
|
||||
recs := make([]store.FederationPeer, 0, len(s.peers))
|
||||
for _, p := range s.peers {
|
||||
recs = append(recs, p.rec)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
results := make(map[string]checkResult, len(recs))
|
||||
var mu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
for _, rec := range recs {
|
||||
wg.Go(func() {
|
||||
res := s.check(ctx, rec)
|
||||
mu.Lock()
|
||||
results[rec.Name] = res
|
||||
mu.Unlock()
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
s.apply(ctx, results)
|
||||
}
|
||||
|
||||
// apply records checks' results: each peer's health always, and its models
|
||||
// when it said them, rebuilding the catalog once if any changed.
|
||||
func (s *Service) apply(ctx context.Context, results map[string]checkResult) {
|
||||
s.op.Lock()
|
||||
defer s.op.Unlock()
|
||||
|
||||
type change struct {
|
||||
name, cluster string
|
||||
models []PeerModel
|
||||
}
|
||||
var changes []change
|
||||
s.mu.Lock()
|
||||
for name, res := range results {
|
||||
p, ok := s.peers[name]
|
||||
if !ok {
|
||||
continue // removed while it was being checked
|
||||
}
|
||||
if p.health.Up != res.health.Up {
|
||||
if res.health.Up {
|
||||
s.log.Info("federation peer is up", "peer", name, "latency", res.health.Latency)
|
||||
} else {
|
||||
s.log.Warn("federation peer is down; its models are not sent requests", "peer", name, "err", res.health.Error)
|
||||
}
|
||||
}
|
||||
p.health = res.health
|
||||
if res.err != nil {
|
||||
continue // its list is kept as it was: down is not empty
|
||||
}
|
||||
next := s.keepReferenced(p, res.models)
|
||||
if res.cluster != p.rec.Cluster || !slices.Equal(next, p.models) {
|
||||
p.models, p.rec.Cluster, p.rec.SyncedAt = next, res.cluster, time.Now()
|
||||
changes = append(changes, change{name, res.cluster, next})
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
if len(changes) == 0 {
|
||||
return
|
||||
}
|
||||
for _, c := range changes {
|
||||
b, _ := json.Marshal(c.models)
|
||||
if err := s.store.SetFederationPeerModels(ctx, c.name, c.cluster, b, time.Now()); err != nil {
|
||||
s.log.Warn("could not keep a federation peer's models", "peer", c.name, "err", err)
|
||||
}
|
||||
s.log.Info("federation peer's models changed", "peer", c.name, "models", len(c.models))
|
||||
}
|
||||
if err := s.catalog.Refresh(ctx); err != nil {
|
||||
s.log.Warn("the catalog could not be rebuilt after a peer's models changed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// checkResult is one check of one peer.
|
||||
type checkResult struct {
|
||||
health Health
|
||||
cluster string
|
||||
models []PeerModel
|
||||
// err is set when the models could not be read; health says the peer
|
||||
// is down then too.
|
||||
err error
|
||||
}
|
||||
|
||||
// check asks a peer whether it is ready, timed, and what it offers this
|
||||
// cluster's key.
|
||||
func (s *Service) check(ctx context.Context, rec store.FederationPeer) checkResult {
|
||||
ctx, cancel := context.WithTimeout(ctx, s.cfg.Timeout)
|
||||
defer cancel()
|
||||
res := checkResult{health: Health{Checked: time.Now()}}
|
||||
fail := func(err error) checkResult {
|
||||
res.health.Up, res.health.Error, res.err = false, err.Error(), err
|
||||
return res
|
||||
}
|
||||
|
||||
// /ready first, and timed: it is open, cheap, and says whether the peer
|
||||
// has anything of its own to serve with.
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rec.URL+"/ready", nil)
|
||||
if err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
start := time.Now()
|
||||
resp, err := s.cfg.Client.Do(req)
|
||||
if err != nil {
|
||||
return fail(fmt.Errorf("the peer did not answer: %w", err))
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<12))
|
||||
resp.Body.Close()
|
||||
res.health.Latency = time.Since(start)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fail(fmt.Errorf("the peer is not ready (%s): it has nothing of its own to serve with", resp.Status))
|
||||
}
|
||||
|
||||
key := ""
|
||||
if rec.KeyFile != "" {
|
||||
b, err := secrets.Read(ctx, rec.KeyFile)
|
||||
if err != nil {
|
||||
return fail(fmt.Errorf("the key for the peer could not be read: %w", err))
|
||||
}
|
||||
key = strings.TrimSpace(string(b))
|
||||
}
|
||||
req, err = http.NewRequestWithContext(ctx, http.MethodGet, rec.URL+"/v1/models", nil)
|
||||
if err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
if key != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+key)
|
||||
}
|
||||
resp, err = s.cfg.Client.Do(req)
|
||||
if err != nil {
|
||||
return fail(fmt.Errorf("the peer's models could not be read: %w", err))
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
switch {
|
||||
case resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden:
|
||||
return fail(fmt.Errorf("the peer refused this cluster's key (%s): check the key file", resp.Status))
|
||||
case resp.StatusCode != http.StatusOK:
|
||||
return fail(fmt.Errorf("the peer's models could not be read: %s", resp.Status))
|
||||
}
|
||||
cluster := resp.Header.Get(ClusterHeader)
|
||||
if cluster == "" {
|
||||
return fail(errors.New("the peer does not say its cluster id: is it an llmgw gateway of this version or later?"))
|
||||
}
|
||||
var list struct {
|
||||
Data []struct {
|
||||
ID string `json:"id"`
|
||||
Description string `json:"description"`
|
||||
ContextLength int `json:"context_length"`
|
||||
MaxOutput int `json:"max_output_tokens"`
|
||||
Capabilities catalog.Capabilities `json:"capabilities"`
|
||||
AliasOf string `json:"alias_of"`
|
||||
Peer string `json:"peer"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&list); err != nil {
|
||||
return fail(fmt.Errorf("the peer's model list is not what an llmgw gateway answers: %w", err))
|
||||
}
|
||||
for _, m := range list.Data {
|
||||
// Not an alias, which is a second name for a model already here,
|
||||
// nor a model the peer itself has from a peer of its own: federating
|
||||
// those would grow a name a hop, A/B/A/..., each time two clusters
|
||||
// read each other's lists.
|
||||
if m.AliasOf != "" || m.Peer != "" || !validModelID(m.ID) {
|
||||
continue
|
||||
}
|
||||
res.models = append(res.models, PeerModel{ID: m.ID, Description: m.Description,
|
||||
ContextLength: m.ContextLength, MaxOutput: m.MaxOutput, Capabilities: m.Capabilities})
|
||||
}
|
||||
slices.SortFunc(res.models, func(a, b PeerModel) int { return strings.Compare(a.ID, b.ID) })
|
||||
res.cluster = cluster
|
||||
res.health.Up = true
|
||||
return res
|
||||
}
|
||||
|
||||
// validModelID is a model name this cluster can carry: printable, no spaces,
|
||||
// and short enough to be a name.
|
||||
func validModelID(id string) bool {
|
||||
if id == "" || len(id) > 200 {
|
||||
return false
|
||||
}
|
||||
return !strings.ContainsFunc(id, func(r rune) bool { return r <= ' ' || r == 0x7f })
|
||||
}
|
||||
|
||||
// Run checks every peer at once each interval until ctx ends, the first
|
||||
// time straight away.
|
||||
func (s *Service) Run(ctx context.Context) {
|
||||
t := time.NewTicker(s.cfg.Interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
s.SyncAll(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Status is a peer as `llmgw ctl federation list` shows it.
|
||||
type Status struct {
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
KeyFile string `json:"key_file,omitempty"`
|
||||
SendTags bool `json:"send_tags"`
|
||||
// Cluster is the peer's own cluster id.
|
||||
Cluster string `json:"cluster,omitempty"`
|
||||
Health
|
||||
// LatencyMs is Latency in milliseconds, for reading.
|
||||
LatencyMs float64 `json:"latency_ms"`
|
||||
// SyncedAt is when its models last changed here.
|
||||
SyncedAt time.Time `json:"synced_at,omitempty"`
|
||||
Models []ModelStatus `json:"models"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
CreatedBy string `json:"created_by,omitempty"`
|
||||
}
|
||||
|
||||
// ModelStatus is one of a peer's models: its name here, the peer's own,
|
||||
// and whether the peer has stopped offering it.
|
||||
type ModelStatus struct {
|
||||
Name string `json:"name"`
|
||||
Upstream string `json:"upstream"`
|
||||
Withdrawn bool `json:"withdrawn,omitempty"`
|
||||
}
|
||||
|
||||
// Status is one peer; false when there is none of the name.
|
||||
func (s *Service) Status(name string) (Status, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
p, ok := s.peers[name]
|
||||
if !ok {
|
||||
return Status{}, false
|
||||
}
|
||||
return statusOf(p), true
|
||||
}
|
||||
|
||||
// List is every peer, by name.
|
||||
func (s *Service) List() []Status {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := make([]Status, 0, len(s.peers))
|
||||
for _, n := range s.namesLocked() {
|
||||
out = append(out, statusOf(s.peers[n]))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func statusOf(p *peer) Status {
|
||||
st := Status{Name: p.rec.Name, URL: p.rec.URL, KeyFile: p.rec.KeyFile, SendTags: p.rec.SendTags,
|
||||
Cluster: p.rec.Cluster, Health: p.health, LatencyMs: float64(p.health.Latency.Microseconds()) / 1000,
|
||||
SyncedAt: p.rec.SyncedAt, CreatedAt: p.rec.CreatedAt, CreatedBy: p.rec.CreatedBy,
|
||||
Models: make([]ModelStatus, 0, len(p.models))}
|
||||
for _, m := range p.models {
|
||||
st.Models = append(st.Models, ModelStatus{Name: p.rec.Name + "/" + m.ID, Upstream: m.ID, Withdrawn: m.Withdrawn})
|
||||
}
|
||||
return st
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package federation
|
||||
|
||||
import "github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
var (
|
||||
upDesc = prometheus.NewDesc("llmgw_federation_peer_up",
|
||||
"Whether a federated peer answered its last check: 1 up, 0 down and not sent requests.",
|
||||
[]string{"peer"}, nil)
|
||||
latencyDesc = prometheus.NewDesc("llmgw_federation_peer_latency_seconds",
|
||||
"How long a federated peer took to answer /ready at its last check.",
|
||||
[]string{"peer"}, nil)
|
||||
modelsDesc = prometheus.NewDesc("llmgw_federation_peer_models",
|
||||
"How many models a federated peer offers this cluster.",
|
||||
[]string{"peer"}, nil)
|
||||
)
|
||||
|
||||
// Describe and Collect publish each peer's health beside the traffic it
|
||||
// takes, so "overflow went to the other site" and "the other site was slow"
|
||||
// are on one chart.
|
||||
func (s *Service) Describe(ch chan<- *prometheus.Desc) {
|
||||
ch <- upDesc
|
||||
ch <- latencyDesc
|
||||
ch <- modelsDesc
|
||||
}
|
||||
|
||||
func (s *Service) Collect(ch chan<- prometheus.Metric) {
|
||||
for _, st := range s.List() {
|
||||
up := 0.0
|
||||
if st.Up {
|
||||
up = 1
|
||||
}
|
||||
offered := 0
|
||||
for _, m := range st.Models {
|
||||
if !m.Withdrawn {
|
||||
offered++
|
||||
}
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(upDesc, prometheus.GaugeValue, up, st.Name)
|
||||
ch <- prometheus.MustNewConstMetric(latencyDesc, prometheus.GaugeValue, st.Latency.Seconds(), st.Name)
|
||||
ch <- prometheus.MustNewConstMetric(modelsDesc, prometheus.GaugeValue, float64(offered), st.Name)
|
||||
}
|
||||
}
|
||||
@@ -51,6 +51,9 @@ type Service struct {
|
||||
// SetCheck.
|
||||
check func(context.Context, catalog.Model) error
|
||||
|
||||
// federated are the models federated clusters offer; see SetFederated.
|
||||
federated func() []catalog.Model
|
||||
|
||||
// mu serialises rebuilds. They are rare — a person adding a model — and two
|
||||
// at once could apply catalogs in either order, leaving the one in force
|
||||
// disagreeing with the database that outlives it.
|
||||
@@ -128,7 +131,7 @@ func (s *Service) Add(ctx context.Context, m catalog.Model, by string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := append([]catalog.Model{}, s.declared...)
|
||||
next := s.base(stored)
|
||||
for _, e := range stored {
|
||||
if e.Name == m.Name {
|
||||
continue // this call replaces it
|
||||
@@ -174,7 +177,7 @@ func (s *Service) Remove(ctx context.Context, name string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := append([]catalog.Model{}, s.declared...)
|
||||
next := s.base(stored)
|
||||
found := false
|
||||
for _, e := range stored {
|
||||
if e.Name == name {
|
||||
@@ -371,7 +374,7 @@ func (s *Service) update(ctx context.Context, name, by, what string, change func
|
||||
if err != nil {
|
||||
return catalog.Model{}, err
|
||||
}
|
||||
next := append([]catalog.Model{}, s.declared...)
|
||||
next := s.base(stored)
|
||||
found := false
|
||||
for _, e := range stored {
|
||||
if e.Name == name {
|
||||
@@ -416,12 +419,19 @@ func (s *Service) Load(ctx context.Context) (*catalog.Catalog, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
return s.load(ctx)
|
||||
}
|
||||
|
||||
// load is Load, with the lock held.
|
||||
func (s *Service) load(ctx context.Context) (*catalog.Catalog, error) {
|
||||
stored, err := s.storedModels(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
next := append([]catalog.Model{}, s.declared...)
|
||||
// The federated models before the stored ones, so a stored model that
|
||||
// falls back to a peer's is checked against a catalog that has it.
|
||||
next := s.base(stored)
|
||||
for _, m := range stored {
|
||||
// Checked one at a time and against what is already accepted, so one
|
||||
// bad model cannot take a good one with it.
|
||||
@@ -459,6 +469,82 @@ func (s *Service) storedModels(ctx context.Context) ([]catalog.Model, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SetFederated supplies the models federated clusters offer, which join the
|
||||
// catalog after the declared ones and before those added at runtime.
|
||||
//
|
||||
// A function rather than a list, read at every rebuild, because the list
|
||||
// changes whenever a peer's does; Refresh is how federation says it has.
|
||||
func (s *Service) SetFederated(f func() []catalog.Model) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.federated = f
|
||||
}
|
||||
|
||||
// Refresh rebuilds the catalog and puts it in force: after a peer's models
|
||||
// changed. Built as at startup, so a stored model that no longer makes a
|
||||
// valid catalog is skipped and said rather than taking the rest with it.
|
||||
func (s *Service) Refresh(ctx context.Context) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
built, err := s.load(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.apply.ReplaceCatalog(ctx, built)
|
||||
return nil
|
||||
}
|
||||
|
||||
// base is the declared models, then the federated ones whose names nothing
|
||||
// declared or added at runtime already uses.
|
||||
//
|
||||
// A peer's model is named PEER/MODEL, which a local model could also be
|
||||
// called — Hugging Face ids look just like that. The local one wins, for the
|
||||
// reason configuration wins over the database: a request for a model this
|
||||
// cluster runs must not leave it because a peer started offering a name.
|
||||
func (s *Service) base(stored []catalog.Model) []catalog.Model {
|
||||
next := append([]catalog.Model{}, s.declared...)
|
||||
if s.federated == nil {
|
||||
return next
|
||||
}
|
||||
taken := map[string]bool{}
|
||||
for _, list := range [][]catalog.Model{s.declared, stored} {
|
||||
for _, m := range list {
|
||||
taken[m.Name] = true
|
||||
for _, a := range m.Aliases {
|
||||
taken[a] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, f := range s.federated() {
|
||||
if taken[f.Name] {
|
||||
s.log.Warn("a federated model has the name of a model of this cluster's; not offered",
|
||||
"model", f.Name)
|
||||
continue
|
||||
}
|
||||
next = append(next, f)
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
// SetFallbacks replaces the models a model added at runtime falls back to
|
||||
// when it has no capacity: other models of the catalog, a federated peer's
|
||||
// among them.
|
||||
func (s *Service) SetFallbacks(ctx context.Context, name string, fallbacks []string, by string) (catalog.Model, error) {
|
||||
final, err := s.update(ctx, name, by, "change its fallbacks", func(m *catalog.Model) error {
|
||||
for _, f := range fallbacks {
|
||||
if f == m.Name {
|
||||
return fmt.Errorf("model %q cannot fall back to itself", name)
|
||||
}
|
||||
}
|
||||
m.Fallbacks = fallbacks
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
s.log.Info("model fallbacks set", "model", name, "fallbacks", fallbacks, "by", by)
|
||||
}
|
||||
return final, err
|
||||
}
|
||||
|
||||
// SetCheck runs before a model is added, and refuses it by returning an error.
|
||||
// Used to catch weights the engine cannot load, which otherwise are placed and
|
||||
// then fail on every node they are tried on.
|
||||
|
||||
Reference in new issue
Block a user